Merge branch 'dev' into 'main'

dev

See merge request oscar.krause/fastapi-dls!48
This commit is contained in:
Oscar Krause 2025-03-17 14:27:23 +01:00
commit d28e123008
10 changed files with 137 additions and 205 deletions

View File

@ -2,7 +2,7 @@ Package: fastapi-dls
Version: 0.0 Version: 0.0
Architecture: all Architecture: all
Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de Maintainer: Oscar Krause oscar.krause@collinwebdesigns.de
Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-josepy, python3-sqlalchemy, python3-pycryptodome, python3-markdown, uvicorn, openssl Depends: python3, python3-fastapi, python3-uvicorn, python3-dotenv, python3-dateutil, python3-josepy, python3-sqlalchemy, python3-cryptography, python3-markdown, uvicorn, openssl
Recommends: curl Recommends: curl
Installed-Size: 10240 Installed-Size: 10240
Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls Homepage: https://git.collinwebdesigns.de/oscar.krause/fastapi-dls

View File

@ -1,8 +1,8 @@
# https://packages.debian.org/hu/ # https://packages.debian.org/hu/
fastapi==0.92.0 fastapi==0.92.0
uvicorn[standard]==0.17.6 uvicorn[standard]==0.17.6
python-jose[pycryptodome]==3.3.0 python-jose[cryptography]==3.3.0
pycryptodome==3.11.0 cryptography==38.0.4
python-dateutil==2.8.2 python-dateutil==2.8.2
sqlalchemy==1.4.46 sqlalchemy==1.4.46
markdown==3.4.1 markdown==3.4.1

View File

@ -1,8 +1,8 @@
# https://packages.ubuntu.com # https://packages.ubuntu.com
fastapi==0.101.0 fastapi==0.101.0
uvicorn[standard]==0.27.1 uvicorn[standard]==0.27.1
python-jose[pycryptodome]==3.3.0 python-jose[cryptography]==3.3.0
pycryptodome==3.20.0 cryptography==41.0.7
python-dateutil==2.8.2 python-dateutil==2.8.2
sqlalchemy==1.4.50 sqlalchemy==1.4.50
markdown==3.5.2 markdown==3.5.2

View File

@ -1,8 +1,8 @@
# https://packages.ubuntu.com # https://packages.ubuntu.com
fastapi==0.110.3 fastapi==0.110.3
uvicorn[standard]==0.30.3 uvicorn[standard]==0.30.3
python-jose[pycryptodome]==3.3.0 python-jose[cryptography]==3.3.0
pycryptodome==3.20.0 cryptography==42.0.5
python-dateutil==2.9.0 python-dateutil==2.9.0
sqlalchemy==2.0.32 sqlalchemy==2.0.32
markdown==3.6 markdown==3.6

View File

@ -8,7 +8,7 @@ pkgdesc='NVIDIA DLS server implementation with FastAPI'
arch=('any') arch=('any')
url='https://git.collinwebdesigns.de/oscar.krause/fastapi-dls' url='https://git.collinwebdesigns.de/oscar.krause/fastapi-dls'
license=('MIT') license=('MIT')
depends=('python' 'python-jose' 'python-starlette' 'python-httpx' 'python-fastapi' 'python-dotenv' 'python-dateutil' 'python-sqlalchemy' 'python-pycryptodome' 'uvicorn' 'python-markdown' 'openssl') depends=('python' 'python-jose' 'python-starlette' 'python-httpx' 'python-fastapi' 'python-dotenv' 'python-dateutil' 'python-sqlalchemy' 'python-cryptography' 'uvicorn' 'python-markdown' 'openssl')
provider=("$pkgname") provider=("$pkgname")
install="$pkgname.install" install="$pkgname.install"
backup=('etc/default/fastapi-dls') backup=('etc/default/fastapi-dls')

View File

@ -21,7 +21,7 @@ from starlette.middleware.cors import CORSMiddleware
from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLResponse as HTMLr, Response, RedirectResponse from starlette.responses import StreamingResponse, JSONResponse as JSONr, HTMLResponse as HTMLr, Response, RedirectResponse
from orm import Origin, Lease, init as db_init, migrate from orm import Origin, Lease, init as db_init, migrate
from util import load_key, load_file from util import load_private_key, load_public_key, get_pem, load_file
# Load variables # Load variables
load_dotenv('../version.env') load_dotenv('../version.env')
@ -42,8 +42,8 @@ DLS_PORT = int(env('DLS_PORT', '443'))
SITE_KEY_XID = str(env('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000')) SITE_KEY_XID = str(env('SITE_KEY_XID', '00000000-0000-0000-0000-000000000000'))
INSTANCE_REF = str(env('INSTANCE_REF', '10000000-0000-0000-0000-000000000001')) INSTANCE_REF = str(env('INSTANCE_REF', '10000000-0000-0000-0000-000000000001'))
ALLOTMENT_REF = str(env('ALLOTMENT_REF', '20000000-0000-0000-0000-000000000001')) ALLOTMENT_REF = str(env('ALLOTMENT_REF', '20000000-0000-0000-0000-000000000001'))
INSTANCE_KEY_RSA = load_key(str(env('INSTANCE_KEY_RSA', join(dirname(__file__), 'cert/instance.private.pem')))) INSTANCE_KEY_RSA = load_private_key(str(env('INSTANCE_KEY_RSA', join(dirname(__file__), 'cert/instance.private.pem'))))
INSTANCE_KEY_PUB = load_key(str(env('INSTANCE_KEY_PUB', join(dirname(__file__), 'cert/instance.public.pem')))) INSTANCE_KEY_PUB = load_public_key(str(env('INSTANCE_KEY_PUB', join(dirname(__file__), 'cert/instance.public.pem'))))
TOKEN_EXPIRE_DELTA = relativedelta(days=int(env('TOKEN_EXPIRE_DAYS', 1)), hours=int(env('TOKEN_EXPIRE_HOURS', 0))) TOKEN_EXPIRE_DELTA = relativedelta(days=int(env('TOKEN_EXPIRE_DAYS', 1)), hours=int(env('TOKEN_EXPIRE_HOURS', 0)))
LEASE_EXPIRE_DELTA = relativedelta(days=int(env('LEASE_EXPIRE_DAYS', 90)), hours=int(env('LEASE_EXPIRE_HOURS', 0))) LEASE_EXPIRE_DELTA = relativedelta(days=int(env('LEASE_EXPIRE_DAYS', 90)), hours=int(env('LEASE_EXPIRE_HOURS', 0)))
LEASE_RENEWAL_PERIOD = float(env('LEASE_RENEWAL_PERIOD', 0.15)) LEASE_RENEWAL_PERIOD = float(env('LEASE_RENEWAL_PERIOD', 0.15))
@ -51,8 +51,8 @@ LEASE_RENEWAL_DELTA = timedelta(days=int(env('LEASE_EXPIRE_DAYS', 90)), hours=in
CLIENT_TOKEN_EXPIRE_DELTA = relativedelta(years=12) CLIENT_TOKEN_EXPIRE_DELTA = relativedelta(years=12)
CORS_ORIGINS = str(env('CORS_ORIGINS', '')).split(',') if (env('CORS_ORIGINS')) else [f'https://{DLS_URL}'] CORS_ORIGINS = str(env('CORS_ORIGINS', '')).split(',') if (env('CORS_ORIGINS')) else [f'https://{DLS_URL}']
jwt_encode_key = jwk.construct(INSTANCE_KEY_RSA.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) jwt_encode_key = jwk.construct(get_pem(INSTANCE_KEY_RSA), algorithm=ALGORITHMS.RS256)
jwt_decode_key = jwk.construct(INSTANCE_KEY_PUB.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) jwt_decode_key = jwk.construct(get_pem(INSTANCE_KEY_PUB), algorithm=ALGORITHMS.RS256)
# Logging # Logging
LOG_LEVEL = logging.DEBUG if DEBUG else logging.INFO LOG_LEVEL = logging.DEBUG if DEBUG else logging.INFO
@ -264,10 +264,10 @@ async def _client_token():
}, },
"service_instance_public_key_configuration": { "service_instance_public_key_configuration": {
"service_instance_public_key_me": { "service_instance_public_key_me": {
"mod": hex(INSTANCE_KEY_PUB.public_key().n)[2:], "mod": hex(INSTANCE_KEY_PUB.public_numbers().n)[2:],
"exp": int(INSTANCE_KEY_PUB.public_key().e), "exp": int(INSTANCE_KEY_PUB.public_numbers().e),
}, },
"service_instance_public_key_pem": INSTANCE_KEY_PUB.export_key().decode('utf-8'), "service_instance_public_key_pem": get_pem(INSTANCE_KEY_PUB).decode('utf-8'),
"key_retention_mode": "LATEST_ONLY" "key_retention_mode": "LATEST_ONLY"
}, },
} }

View File

@ -11,31 +11,51 @@ def load_file(filename: str) -> bytes:
return content return content
def load_key(filename: str) -> "RsaKey": def load_private_key(filename: str) -> "RSAPrivateKey":
try: from cryptography.hazmat.primitives.serialization import load_pem_private_key
# Crypto | Cryptodome on Debian
from Crypto.PublicKey import RSA
from Crypto.PublicKey.RSA import RsaKey
except ModuleNotFoundError:
from Cryptodome.PublicKey import RSA
from Cryptodome.PublicKey.RSA import RsaKey
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
log.debug(f'Importing RSA-Key from "{filename}"') log.debug(f'Importing RSA-Key from "{filename}"')
return RSA.import_key(extern_key=load_file(filename), passphrase=None)
with open(filename, 'rb') as f:
data = f.read()
return load_pem_private_key(data.strip(), password=None)
def generate_key() -> "RsaKey": def load_public_key(filename: str) -> "RSAPublicKey":
try: from cryptography.hazmat.primitives.serialization import load_pem_public_key
# Crypto | Cryptodome on Debian
from Crypto.PublicKey import RSA log = logging.getLogger(__name__)
from Crypto.PublicKey.RSA import RsaKey log.debug(f'Importing RSA-Key from "{filename}"')
except ModuleNotFoundError:
from Cryptodome.PublicKey import RSA with open(filename, 'rb') as f:
from Cryptodome.PublicKey.RSA import RsaKey data = f.read()
return load_pem_public_key(data.strip())
def get_pem(key) -> bytes | None:
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey
from cryptography.hazmat.primitives import serialization
if isinstance(key, RSAPrivateKey):
return key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption()
)
if isinstance(key, RSAPublicKey):
return key.public_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PublicFormat.SubjectPublicKeyInfo
)
def generate_private_key() -> "RSAPrivateKey":
from cryptography.hazmat.primitives.asymmetric import rsa
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
log.debug(f'Generating RSA-Key') log.debug(f'Generating RSA-Key')
return RSA.generate(bits=2048) return rsa.generate_private_key(public_exponent=65537, key_size=2048)
class NV: class NV:

View File

@ -2,7 +2,75 @@
[[_TOC_]] [[_TOC_]]
# Usefully commands # NLS Docker Stack
- More about Docker Images https://git.collinwebdesigns.de/nvidia/nls
## Appliance
### Configuration data
- Most variables and configs are stored in `/var/lib/docker/volumes/configurations/_data`.
- Config-Variables are in `etc/dls/config/service_env.conf`.
### NLS Logs
Logs are found in `/var/lib/docker/volumes/logs/_data`.
Most interesting logs are:
- `fileInstallation.log`
- `serviceInstance.log`
### File manipulation and copy
- Files can be copied with `docker cp <container-id>:/venv/... /opt/localfile/...`.
- Files can be directly edited via Docker-Volume mounts
- see `df -h` (one is nls, the other postgres container)
```
overlay 16G 11G 5.6G 66% /var/lib/docker/overlay2/<hash>/merged
overlay 16G 11G 5.6G 66% /var/lib/docker/overlay2/<hash>/merged
```
- then you can edit files with e.g. `nano venv/lib/python3.12/site-packages/...`
### Other tools / files
Other tools / files which may can helpful, but not known for what they are used.
- `/etc/dls/config/decryptor/decryptor`
- `/etc/dls/config/site_key_uri.bin`
- `/etc/dls/config/dls_db_password.bin`
## Database
- It's enough to manipulate database licenses. There must not be changed any line of code to bypass licensing
validations.
Valid users are `dls_writer` and `postgres`.
```shell
docker exec -it <dls:pgsql> psql -h localhost -U postgres
```
If you want external access to database, you have to add `ports: [ 5432:5432 ]` to postgres section in
`docker-compose.yml`.
Then you can *exec* into container with `psql` and add a new superuser:
```sql
CREATE
USER admin WITH LOGIN SUPERUSER PASSWORD 'admin';
```
# Logging / Stack Trace
- https://docs.nvidia.com/license-system/latest/nvidia-license-system-user-guide/index.html#troubleshooting-dls-instance
# Nginx
- NGINX uses `/opt/certs/cert.pem` and `/opt/certs/key.pem`
# Usefully commands on Client
## Check licensing status ## Check licensing status
@ -20,167 +88,11 @@ vGPU Software Licensed Product
- NVIDIA Grid Log: `journalctl -u nvidia-gridd -f` - NVIDIA Grid Log: `journalctl -u nvidia-gridd -f`
``` ```
systemd[1]: Started NVIDIA Grid Daemon. systemd: Started NVIDIA Grid Daemon.
nvidia-gridd[2986]: Configuration parameter ( ServerAddress ) not set nvidia-gridd: Configuration parameter ( ServerAddress ) not set
nvidia-gridd[2986]: vGPU Software package (0) nvidia-gridd: vGPU Software package (0)
nvidia-gridd[2986]: Ignore service provider and node-locked licensing nvidia-gridd: Ignore service provider and node-locked licensing
nvidia-gridd[2986]: NLS initialized nvidia-gridd: NLS initialized
nvidia-gridd[2986]: Acquiring license. (Info: license.nvidia.space; NVIDIA RTX Virtual Workstation) nvidia-gridd: Acquiring license. (Info: license.nvidia.space; NVIDIA RTX Virtual Workstation)
nvidia-gridd[2986]: License acquired successfully. (Info: license.nvidia.space, NVIDIA RTX Virtual Workstation; Expiry: 2023-1-29 22:3:0 GMT) nvidia-gridd: License acquired successfully. (Info: license.nvidia.space, NVIDIA RTX Virtual Workstation; Expiry: 2023-1-29 22:3:0 GMT)
``` ```
# Docker DLS-Container File-System
- More about Docker Images https://git.collinwebdesigns.de/nvidia/nls
## Configuration data
Most variables and configs are stored in `/var/lib/docker/volumes/configurations/_data`.
Files can be modified with `docker cp <container-id>:/venv/... /opt/localfile/...` and back.
(May you need to fix permissions with `docker exec -u 0 <container-id> chown nonroot:nonroot /venv/...`)
Config-Variables are in `etc/dls/config/service_env.conf`.
## Site Key Uri - `/etc/dls/config/site_key_uri.bin`
```
base64-content...
```
## DB Password - `/etc/dls/config/dls_db_password.bin`
```
# docker cp -a <container-id>:/etc/dls/config/dls_db_password.bin /tmp/dls_db_password.bin
base64-content...
```
**Decrypt database password**
```
cat dls_db_password.bin | base64 -d > dls_db_password.bin.raw
openssl rsautl -decrypt -inkey /tmp/private-key.pem -in dls_db_password.bin.raw
```
# Docker Postgres-Container
- It's enough to manipulate database licenses. There must not be changed any line of code to bypass licensing
validations.
## Inspect
Valid users are `dls_writer` and `postgres`.
```shell
docker exec -it <dls:pgsql> psql -h localhost -U postgres
```
## External Access
Or you can modify `docker-compose.yaml` to forward Postgres port. To create a superuser for external access, use `docker exec` from above and rund the following:
```sql
CREATE USER admin WITH LOGIN SUPERUSER PASSWORD 'admin';
```
# Dive / Docker image inspector
- `dive dls:appliance`
The source code is stored in `/venv/lib/python3.9/site-packages/nls_*`.
Image-Reference:
```
Tags: (unavailable)
Id: d1c7976a5d2b3681ff6c5a30f8187e4015187a83f3f285ba4a37a45458bd6b98
Digest: sha256:311223c5af7a298ec1104f5dc8c3019bfb0e1f77256dc3d995244ffb295a97
1f
Command:
#(nop) ADD file:c1900d3e3a29c29a743a8da86c437006ec5d2aa873fb24e48033b6bf492bb37b in /
```
# Logging / Stack Trace
- https://docs.nvidia.com/license-system/latest/nvidia-license-system-user-guide/index.html#troubleshooting-dls-instance
**Failed licensing log**
```
{
"activity": 100,
"context": {
"SERVICE_INSTANCE_ID": "b43d6e46-d6d0-4943-8b8d-c66a5f6e0d38",
"SERVICE_INSTANCE_NAME": "DEFAULT_2022-12-14_12:48:30",
"description": "borrow failed: NotFoundError(no pool features found for: NVIDIA RTX Virtual Workstation)",
"event_type": null,
"function_name": "_evt",
"lineno": 54,
"module_name": "nls_dal_lease_dls.event",
"operation_id": "e72a8ca7-34cc-4e11-b80c-273592085a24",
"origin_ref": "3f7f5a50-a26b-425b-8d5e-157f63e72b1c",
"service_name": "nls_services_lease"
},
"detail": {
"oc": {
"license_allotment_xid": "10c4317f-7c4c-11ed-a524-0e4252a7e5f1",
"origin_ref": "3f7f5a50-a26b-425b-8d5e-157f63e72b1c",
"service_instance_xid": "b43d6e46-d6d0-4943-8b8d-c66a5f6e0d38"
},
"operation_id": "e72a8ca7-34cc-4e11-b80c-273592085a24"
},
"id": "0cc9e092-3b92-4652-8d9e-7622ef85dc79",
"metadata": {},
"ts": "2022-12-15T10:25:36.827661Z"
}
{
"activity": 400,
"context": {
"SERVICE_INSTANCE_ID": "b43d6e46-d6d0-4943-8b8d-c66a5f6e0d38",
"SERVICE_INSTANCE_NAME": "DEFAULT_2022-12-14_12:48:30",
"description": "lease_multi_create failed: no pool features found for: NVIDIA RTX Virtual Workstation",
"event_by": "system",
"function_name": "lease_multi_create",
"level": "warning",
"lineno": 157,
"module_name": "nls_services_lease.controllers.lease_multi_controller",
"operation_id": "e72a8ca7-34cc-4e11-b80c-273592085a24",
"service_name": "nls_services_lease"
},
"detail": {
"_msg": "lease_multi_create failed: no pool features found for: NVIDIA RTX Virtual Workstation",
"exec_info": ["NotFoundError", "NotFoundError(no pool features found for: NVIDIA RTX Virtual Workstation)", " File \"/venv/lib/python3.9/site-packages/nls_services_lease/controllers/lease_multi_controller.py\", line 127, in lease_multi_create\n data = _leaseMulti.lease_multi_create(event_args)\n File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 208, in lease_multi_create\n raise e\n File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 184, in lease_multi_create\n self._try_proposals(oc, mlr, results, detail)\n File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 219, in _try_proposals\n lease = self._leases.create(creator)\n File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 230, in create\n features = self._get_features(creator)\n File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 148, in _get_features\n self._explain_not_available(cur, creator)\n File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 299, in _explain_not_available\n raise NotFoundError(f'no pool features found for: {lcc.product_name}')\n"],
"operation_id": "e72a8ca7-34cc-4e11-b80c-273592085a24"
},
"id": "282801b9-d612-40a5-9145-b56d8e420dac",
"metadata": {},
"ts": "2022-12-15T10:25:36.831673Z"
}
```
**Stack Trace**
```
"NotFoundError", "NotFoundError(no pool features found for: NVIDIA RTX Virtual Workstation)", " File \"/venv/lib/python3.9/site-packages/nls_services_lease/controllers/lease_multi_controller.py\", line 127, in lease_multi_create
data = _leaseMulti.lease_multi_create(event_args)
File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 208, in lease_multi_create
raise e
File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 184, in lease_multi_create
self._try_proposals(oc, mlr, results, detail)
File \"/venv/lib/python3.9/site-packages/nls_core_lease/lease_multi.py\", line 219, in _try_proposals
lease = self._leases.create(creator)
File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 230, in create
features = self._get_features(creator)
File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 148, in _get_features
self._explain_not_available(cur, creator)
File \"/venv/lib/python3.9/site-packages/nls_dal_lease_dls/leases.py\", line 299, in _explain_not_available
raise NotFoundError(f'no pool features found for: {lcc.product_name}')
"
```
# Nginx
- NGINX uses `/opt/certs/cert.pem` and `/opt/certs/key.pem`

View File

@ -1,7 +1,7 @@
fastapi==0.115.8 fastapi==0.115.8
uvicorn[standard]==0.34.0 uvicorn[standard]==0.34.0
python-jose==3.4.0 python-jose[cryptography]==3.4.0
pycryptodome==3.21.0 cryptography==44.0.2
python-dateutil==2.8.2 python-dateutil==2.8.2
sqlalchemy==2.0.38 sqlalchemy==2.0.38
markdown==3.7 markdown==3.7

View File

@ -16,7 +16,7 @@ sys.path.append('../')
sys.path.append('../app') sys.path.append('../app')
from app import main from app import main
from app.util import load_key from util import load_private_key, load_public_key, get_pem
client = TestClient(main.app) client = TestClient(main.app)
@ -25,11 +25,11 @@ ORIGIN_REF, ALLOTMENT_REF, SECRET = str(uuid4()), '20000000-0000-0000-0000-00000
# INSTANCE_KEY_RSA = generate_key() # INSTANCE_KEY_RSA = generate_key()
# INSTANCE_KEY_PUB = INSTANCE_KEY_RSA.public_key() # INSTANCE_KEY_PUB = INSTANCE_KEY_RSA.public_key()
INSTANCE_KEY_RSA = load_key(str(join(dirname(__file__), '../app/cert/instance.private.pem'))) INSTANCE_KEY_RSA = load_private_key(str(join(dirname(__file__), '../app/cert/instance.private.pem')))
INSTANCE_KEY_PUB = load_key(str(join(dirname(__file__), '../app/cert/instance.public.pem'))) INSTANCE_KEY_PUB = load_public_key(str(join(dirname(__file__), '../app/cert/instance.public.pem')))
jwt_encode_key = jwk.construct(INSTANCE_KEY_RSA.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) jwt_encode_key = jwk.construct(get_pem(INSTANCE_KEY_RSA), algorithm=ALGORITHMS.RS256)
jwt_decode_key = jwk.construct(INSTANCE_KEY_PUB.export_key().decode('utf-8'), algorithm=ALGORITHMS.RS256) jwt_decode_key = jwk.construct(get_pem(INSTANCE_KEY_PUB), algorithm=ALGORITHMS.RS256)
def __bearer_token(origin_ref: str) -> str: def __bearer_token(origin_ref: str) -> str: