diff --git a/.DEBIAN/control b/.DEBIAN/control index c2f6ccd..b58c41c 100644 --- a/.DEBIAN/control +++ b/.DEBIAN/control @@ -2,7 +2,7 @@ Package: fastapi-dls Version: 0.0 Architecture: all Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de -Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-josepy, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl +Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-josepy, python3-sqlalchemy, python3-cryptography, python3-markdown, uvicorn, openssl Recommends: curl Installed-Size: 10240 Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls diff --git a/.DEBIAN/requirements-bookworm-12.txt b/.DEBIAN/requirements-bookworm-12.txt index 223c64c..c3fe52e 100644 --- a/.DEBIAN/requirements-bookworm-12.txt +++ b/.DEBIAN/requirements-bookworm-12.txt @@ -1,8 +1,8 @@ # https://packages.debian.org/hu/ fastapi==0.92.0 uvicorn[standard]==0.17.6 -python-jose[pycryptodome]==3.3.0 -pycryptodome==3.11.0 +python-jose[cryptography]==3.3.0 +cryptography==38.0.4 python-dateutil==2.8.2 sqlalchemy==1.4.46 markdown==3.4.1 diff --git a/.DEBIAN/requirements-ubuntu-24.04.txt b/.DEBIAN/requirements-ubuntu-24.04.txt index 7cb653b..0ba3025 100644 --- a/.DEBIAN/requirements-ubuntu-24.04.txt +++ b/.DEBIAN/requirements-ubuntu-24.04.txt @@ -1,8 +1,8 @@ # https://packages.ubuntu.com fastapi==0.101.0 uvicorn[standard]==0.27.1 -python-jose[pycryptodome]==3.3.0 -pycryptodome==3.20.0 +python-jose[cryptography]==3.3.0 +cryptography==41.0.7 python-dateutil==2.8.2 sqlalchemy==1.4.50 markdown==3.5.2 diff --git a/.DEBIAN/requirements-ubuntu-24.10.txt b/.DEBIAN/requirements-ubuntu-24.10.txt index 7a65314..59f9361 100644 --- a/.DEBIAN/requirements-ubuntu-24.10.txt +++ b/.DEBIAN/requirements-ubuntu-24.10.txt @@ -1,8 +1,8 @@ # https://packages.ubuntu.com fastapi==0.110.3 uvicorn[standard]==0.30.3 -python-jose[pycryptodome]==3.3.0 -pycryptodome==3.20.0 +python-jose[cryptography]==3.3.0 +cryptography==42.0.5 python-dateutil==2.9.0 sqlalchemy==2.0.32 markdown==3.6 diff --git a/.PKGBUILD/PKGBUILD b/.PKGBUILD/PKGBUILD index 09f606b..bee4a12 100644 --- a/.PKGBUILD/PKGBUILD +++ b/.PKGBUILD/PKGBUILD @@ -8,7 +8,7 @@ pkgdesc='NVIDIA DLS server implementation with FastAPI' arch=('any') url='https://git.collinwebdesigns.de/oscar.krause/fastapi-dls' license=('MIT') -depends=('python' 'python-jose' 'python-starlette' 'python-httpx' 'python-fastapi' 'python-dotenv' 'python-dateutil' 'python-sqlalchemy' 'python-pycryptodome' 'uvicorn' 'python-markdown' 'openssl') +depends=('python' 'python-jose' 'python-starlette' 'python-httpx' 'python-fastapi' 'python-dotenv' 'python-dateutil' 'python-sqlalchemy' 'python-cryptography' 'uvicorn' 'python-markdown' 'openssl') provider=("$pkgname") install="$pkgname.install" backup=('etc/default/fastapi-dls') diff --git a/app/main.py b/app/main.py index 850a6a2..60f8c60 100644 --- a/app/main.py +++ b/app/main.py @@ -21,7 +21,7 @@ from starlette.middleware.cors import CORSMiddleware from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLResponse as HTMLr, Response, RedirectResponse from orm import Origin, Lease, init as db_init, migrate -from util import load_key, load_file +from util import load_private_key, load_public_key, get_pem, load_file # Load variables load_dotenv('../version.env') @@ -42,8 +42,8 @@ DLS_PORT = int(env('DLS_PORT', '443')) SITE_KEY_XID = str(env('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000')) INSTANCE_REF = str(env('INSTANCE_REF', '10000000-0000-0000-0000-000000000001')) ALLOTMENT_REF = str(env('ALLOTMENT_REF', '20000000-0000-0000-0000-000000000001')) -INSTANCE_KEY_RSA = load_key(str(env('INSTANCE_KEY_RSA', join(dirname(__file__), 'cert/instance.private.pem')))) -INSTANCE_KEY_PUB = load_key(str(env('INSTANCE_KEY_PUB', join(dirname(__file__), 'cert/instance.public.pem')))) +INSTANCE_KEY_RSA = load_private_key(str(env('INSTANCE_KEY_RSA', join(dirname(__file__), 'cert/instance.private.pem')))) +INSTANCE_KEY_PUB = load_public_key(str(env('INSTANCE_KEY_PUB', join(dirname(__file__), 'cert/instance.public.pem')))) TOKEN_EXPIRE_DELTA = relativedelta(days=int(env('TOKEN_EXPIRE_DAYS', 1)), hours=int(env('TOKEN_EXPIRE_HOURS', 0))) LEASE_EXPIRE_DELTA = relativedelta(days=int(env('LEASE_EXPIRE_DAYS', 90)), hours=int(env('LEASE_EXPIRE_HOURS', 0))) LEASE_RENEWAL_PERIOD = float(env('LEASE_RENEWAL_PERIOD', 0.15)) @@ -51,8 +51,8 @@ LEASE_RENEWAL_DELTA = timedelta(days=int(env('LEASE_EXPIRE_DAYS', 90)), hours=in CLIENT_TOKEN_EXPIRE_DELTA = relativedelta(years=12) CORS_ORIGINS = str(env('CORS_ORIGINS', '')).split(',') if (env('CORS_ORIGINS')) else [f'https://{DLS_URL}'] -jwt_encode_key = jwk.construct(INSTANCE_KEY_RSA.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) -jwt_decode_key = jwk.construct(INSTANCE_KEY_PUB.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) +jwt_encode_key = jwk.construct(get_pem(INSTANCE_KEY_RSA), algorithm=ALGORITHMS.RS256) +jwt_decode_key = jwk.construct(get_pem(INSTANCE_KEY_PUB), algorithm=ALGORITHMS.RS256) # Logging LOG_LEVEL = logging.DEBUG if DEBUG else logging.INFO @@ -264,10 +264,10 @@ async def _client_token(): }, "service_instance_public_key_configuration": { "service_instance_public_key_me": { - "mod": hex(INSTANCE_KEY_PUB.public_key().n)[2:], - "exp": int(INSTANCE_KEY_PUB.public_key().e), + "mod": hex(INSTANCE_KEY_PUB.public_numbers().n)[2:], + "exp": int(INSTANCE_KEY_PUB.public_numbers().e), }, - "service_instance_public_key_pem": INSTANCE_KEY_PUB.export_key().decode('utf-8'), + "service_instance_public_key_pem": get_pem(INSTANCE_KEY_PUB).decode('utf-8'), "key_retention_mode": "LATEST_ONLY" }, } diff --git a/app/util.py b/app/util.py index b5b1ff1..f2b1be4 100644 --- a/app/util.py +++ b/app/util.py @@ -11,31 +11,51 @@ def load_file(filename: str) -> bytes: return content -def load_key(filename: str) -> "RsaKey": - try: - # Crypto | Cryptodome on Debian - from Crypto.PublicKey import RSA - from Crypto.PublicKey.RSA import RsaKey - except ModuleNotFoundError: - from Cryptodome.PublicKey import RSA - from Cryptodome.PublicKey.RSA import RsaKey +def load_private_key(filename: str) -> "RSAPrivateKey": + from cryptography.hazmat.primitives.serialization import load_pem_private_key log = logging.getLogger(__name__) log.debug(f'Importing RSA-Key from "{filename}"') - return RSA.import_key(extern_key=load_file(filename), passphrase=None) + + with open(filename, 'rb') as f: + data = f.read() + return load_pem_private_key(data.strip(), password=None) -def generate_key() -> "RsaKey": - try: - # Crypto | Cryptodome on Debian - from Crypto.PublicKey import RSA - from Crypto.PublicKey.RSA import RsaKey - except ModuleNotFoundError: - from Cryptodome.PublicKey import RSA - from Cryptodome.PublicKey.RSA import RsaKey +def load_public_key(filename: str) -> "RSAPublicKey": + from cryptography.hazmat.primitives.serialization import load_pem_public_key + + log = logging.getLogger(__name__) + log.debug(f'Importing RSA-Key from "{filename}"') + + with open(filename, 'rb') as f: + data = f.read() + return load_pem_public_key(data.strip()) + + +def get_pem(key) -> bytes | None: + from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey + from cryptography.hazmat.primitives import serialization + + if isinstance(key, RSAPrivateKey): + return key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption() + ) + if isinstance(key, RSAPublicKey): + return key.public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo + ) + + +def generate_private_key() -> "RSAPrivateKey": + from cryptography.hazmat.primitives.asymmetric import rsa + log = logging.getLogger(__name__) log.debug(f'Generating RSA-Key') - return RSA.generate(bits=2048) + return rsa.generate_private_key(public_exponent=65537, key_size=2048) class NV: diff --git a/doc/Reverse Engineering Notes.md b/doc/Reverse Engineering Notes.md index fd1f67f..ae1ba25 100644 --- a/doc/Reverse Engineering Notes.md +++ b/doc/Reverse Engineering Notes.md @@ -2,7 +2,75 @@ [[_TOC_]] -# Usefully commands +# NLS Docker Stack + +- More about Docker Images https://git.collinwebdesigns.de/nvidia/nls + +## Appliance + +### Configuration data + +- Most variables and configs are stored in `/var/lib/docker/volumes/configurations/_data`. +- Config-Variables are in `etc/dls/config/service_env.conf`. + +### NLS Logs + +Logs are found in `/var/lib/docker/volumes/logs/_data`. + +Most interesting logs are: + +- `fileInstallation.log` +- `serviceInstance.log` + +### File manipulation and copy + +- Files can be copied with `docker cp :/venv/... /opt/localfile/...`. +- Files can be directly edited via Docker-Volume mounts + - see `df -h` (one is nls, the other postgres container) + ``` + overlay 16G 11G 5.6G 66% /var/lib/docker/overlay2//merged + overlay 16G 11G 5.6G 66% /var/lib/docker/overlay2//merged + ``` + - then you can edit files with e.g. `nano venv/lib/python3.12/site-packages/...` + +### Other tools / files + +Other tools / files which may can helpful, but not known for what they are used. + +- `/etc/dls/config/decryptor/decryptor` +- `/etc/dls/config/site_key_uri.bin` +- `/etc/dls/config/dls_db_password.bin` + +## Database + +- It's enough to manipulate database licenses. There must not be changed any line of code to bypass licensing + validations. + +Valid users are `dls_writer` and `postgres`. + +```shell +docker exec -it psql -h localhost -U postgres +``` + +If you want external access to database, you have to add `ports: [ 5432:5432 ]` to postgres section in +`docker-compose.yml`. +Then you can *exec* into container with `psql` and add a new superuser: + +```sql +CREATE +USER admin WITH LOGIN SUPERUSER PASSWORD 'admin'; +``` + +# Logging / Stack Trace + +- https://docs.nvidia.com/license-system/latest/nvidia-license-system-user-guide/index.html#troubleshooting-dls-instance + + +# Nginx + +- NGINX uses `/opt/certs/cert.pem` and `/opt/certs/key.pem` + +# Usefully commands on Client ## Check licensing status @@ -20,167 +88,11 @@ vGPU Software Licensed Product - NVIDIA Grid Log: `journalctl -u nvidia-gridd -f` ``` -systemd[1]: Started NVIDIA Grid Daemon. -nvidia-gridd[2986]: Configuration parameter ( ServerAddress ) not set -nvidia-gridd[2986]: vGPU Software package (0) -nvidia-gridd[2986]: Ignore service provider and node-locked licensing -nvidia-gridd[2986]: NLS initialized -nvidia-gridd[2986]: Acquiring license. (Info: license.nvidia.space; NVIDIA RTX Virtual Workstation) -nvidia-gridd[2986]: License acquired successfully. (Info: license.nvidia.space, NVIDIA RTX Virtual Workstation; Expiry: 2023-1-29 22:3:0 GMT) +systemd: Started NVIDIA Grid Daemon. +nvidia-gridd: Configuration parameter ( ServerAddress ) not set +nvidia-gridd: vGPU Software package (0) +nvidia-gridd: Ignore service provider and node-locked licensing +nvidia-gridd: NLS initialized +nvidia-gridd: Acquiring license. (Info: license.nvidia.space; NVIDIA RTX Virtual Workstation) +nvidia-gridd: License acquired successfully. (Info: license.nvidia.space, NVIDIA RTX Virtual Workstation; Expiry: 2023-1-29 22:3:0 GMT) ``` - -# Docker DLS-Container File-System - -- More about Docker Images https://git.collinwebdesigns.de/nvidia/nls - -## Configuration data - -Most variables and configs are stored in `/var/lib/docker/volumes/configurations/_data`. - -Files can be modified with `docker cp :/venv/... /opt/localfile/...` and back. -(May you need to fix permissions with `docker exec -u 0 chown nonroot:nonroot /venv/...`) - -Config-Variables are in `etc/dls/config/service_env.conf`. - - -## Site Key Uri - `/etc/dls/config/site_key_uri.bin` - -``` -base64-content... -``` - -## DB Password - `/etc/dls/config/dls_db_password.bin` - -``` -# docker cp -a :/etc/dls/config/dls_db_password.bin /tmp/dls_db_password.bin -base64-content... -``` - -**Decrypt database password** - -``` -cat dls_db_password.bin | base64 -d > dls_db_password.bin.raw -openssl rsautl -decrypt -inkey /tmp/private-key.pem -in dls_db_password.bin.raw -``` - -# Docker Postgres-Container - -- It's enough to manipulate database licenses. There must not be changed any line of code to bypass licensing - validations. - -## Inspect - -Valid users are `dls_writer` and `postgres`. - -```shell -docker exec -it psql -h localhost -U postgres -``` - -## External Access - -Or you can modify `docker-compose.yaml` to forward Postgres port. To create a superuser for external access, use `docker exec` from above and rund the following: - -```sql -CREATE USER admin WITH LOGIN SUPERUSER PASSWORD 'admin'; -``` - -# Dive / Docker image inspector - -- `dive dls:appliance` - -The source code is stored in `/venv/lib/python3.9/site-packages/nls_*`. - -Image-Reference: - -``` -Tags: (unavailable) -Id: d1c7976a5d2b3681ff6c5a30f8187e4015187a83f3f285ba4a37a45458bd6b98 -Digest: sha256:311223c5af7a298ec1104f5dc8c3019bfb0e1f77256dc3d995244ffb295a97 -1f -Command: -#(nop) ADD file:c1900d3e3a29c29a743a8da86c437006ec5d2aa873fb24e48033b6bf492bb37b in / -``` - -# Logging / Stack Trace - -- https://docs.nvidia.com/license-system/latest/nvidia-license-system-user-guide/index.html#troubleshooting-dls-instance - -**Failed licensing log** - -``` -{ - "activity": 100, - "context": { - "SERVICE_INSTANCE_ID": "b43d6e46-d6d0-4943-8b8d-c66a5f6e0d38", - "SERVICE_INSTANCE_NAME": "DEFAULT_2022-12-14_12:48:30", - "description": "borrow failed: NotFoundError(no pool features found for: NVIDIA RTX Virtual Workstation)", - "event_type": null, - "function_name": "_evt", - "lineno": 54, - "module_name": "nls_dal_lease_dls.event", - "operation_id": "e72a8ca7-34cc-4e11-b80c-273592085a24", - "origin_ref": "3f7f5a50-a26b-425b-8d5e-157f63e72b1c", - "service_name": "nls_services_lease" - }, - "detail": { - "oc": { - "license_allotment_xid": "10c4317f-7c4c-11ed-a524-0e4252a7e5f1", - "origin_ref": "3f7f5a50-a26b-425b-8d5e-157f63e72b1c", - "service_instance_xid": "b43d6e46-d6d0-4943-8b8d-c66a5f6e0d38" - }, - "operation_id": "e72a8ca7-34cc-4e11-b80c-273592085a24" - }, - "id": "0cc9e092-3b92-4652-8d9e-7622ef85dc79", - "metadata": {}, - "ts": "2022-12-15T10:25:36.827661Z" -} - -{ - "activity": 400, - "context": { - "SERVICE_INSTANCE_ID": "b43d6e46-d6d0-4943-8b8d-c66a5f6e0d38", - "SERVICE_INSTANCE_NAME": "DEFAULT_2022-12-14_12:48:30", - "description": "lease_multi_create failed: no pool features found for: NVIDIA RTX Virtual Workstation", - "event_by": "system", - "function_name": "lease_multi_create", - "level": "warning", - "lineno": 157, - "module_name": "nls_services_lease.controllers.lease_multi_controller", - "operation_id": "e72a8ca7-34cc-4e11-b80c-273592085a24", - "service_name": "nls_services_lease" - }, - "detail": { - "_msg": "lease_multi_create failed: no pool features found for: NVIDIA RTX Virtual Workstation", - "exec_info": ["NotFoundError", "NotFoundError(no pool features found for: NVIDIA RTX Virtual Workstation)", " File \"/venv/lib/python3.9/site-packages/nls_services_lease/controllers/lease_multi_controller.py\", line 127, in lease_multi_create\n data = _leaseMulti.lease_multi_create(event_args)\n File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 208, in lease_multi_create\n raise e\n File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 184, in lease_multi_create\n self._try_proposals(oc, mlr, results, detail)\n File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 219, in _try_proposals\n lease = self._leases.create(creator)\n File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 230, in create\n features = self._get_features(creator)\n File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 148, in _get_features\n self._explain_not_available(cur, creator)\n File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 299, in _explain_not_available\n raise NotFoundError(f'no pool features found for: {lcc.product_name}')\n"], - "operation_id": "e72a8ca7-34cc-4e11-b80c-273592085a24" - }, - "id": "282801b9-d612-40a5-9145-b56d8e420dac", - "metadata": {}, - "ts": "2022-12-15T10:25:36.831673Z" -} - -``` - -**Stack Trace** - -``` -"NotFoundError", "NotFoundError(no pool features found for: NVIDIA RTX Virtual Workstation)", " File \"/venv/lib/python3.9/site-packages/nls_services_lease/controllers/lease_multi_controller.py\", line 127, in lease_multi_create - data = _leaseMulti.lease_multi_create(event_args) - File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 208, in lease_multi_create - raise e - File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 184, in lease_multi_create - self._try_proposals(oc, mlr, results, detail) - File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 219, in _try_proposals - lease = self._leases.create(creator) - File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 230, in create - features = self._get_features(creator) - File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 148, in _get_features - self._explain_not_available(cur, creator) - File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 299, in _explain_not_available - raise NotFoundError(f'no pool features found for: {lcc.product_name}') -" -``` - -# Nginx - -- NGINX uses `/opt/certs/cert.pem` and `/opt/certs/key.pem` diff --git a/requirements.txt b/requirements.txt index 6faecca..e13bedc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ fastapi==0.115.8 uvicorn[standard]==0.34.0 -python-jose==3.4.0 -pycryptodome==3.21.0 +python-jose[cryptography]==3.4.0 +cryptography==44.0.2 python-dateutil==2.8.2 sqlalchemy==2.0.38 markdown==3.7 diff --git a/test/main.py b/test/main.py index a10f2c7..b63601e 100644 --- a/test/main.py +++ b/test/main.py @@ -16,7 +16,7 @@ sys.path.append('../') sys.path.append('../app') from app import main -from app.util import load_key +from util import load_private_key, load_public_key, get_pem client = TestClient(main.app) @@ -25,11 +25,11 @@ ORIGIN_REF, ALLOTMENT_REF, SECRET = str(uuid4()), '20000000-0000-0000-0000-00000 # INSTANCE_KEY_RSA = generate_key() # INSTANCE_KEY_PUB = INSTANCE_KEY_RSA.public_key() -INSTANCE_KEY_RSA = load_key(str(join(dirname(__file__), '../app/cert/instance.private.pem'))) -INSTANCE_KEY_PUB = load_key(str(join(dirname(__file__), '../app/cert/instance.public.pem'))) +INSTANCE_KEY_RSA = load_private_key(str(join(dirname(__file__), '../app/cert/instance.private.pem'))) +INSTANCE_KEY_PUB = load_public_key(str(join(dirname(__file__), '../app/cert/instance.public.pem'))) -jwt_encode_key = jwk.construct(INSTANCE_KEY_RSA.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) -jwt_decode_key = jwk.construct(INSTANCE_KEY_PUB.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) +jwt_encode_key = jwk.construct(get_pem(INSTANCE_KEY_RSA), algorithm=ALGORITHMS.RS256) +jwt_decode_key = jwk.construct(get_pem(INSTANCE_KEY_PUB), algorithm=ALGORITHMS.RS256) def __bearer_token(origin_ref: str) -> str: