diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c136ce..672f66c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,12 +25,6 @@ repos: language: system types: [python] require_serial: true - - id: biome - name: Biome formatting - entry: biome format --write --files-ignore-unknown=true --no-errors-on-unmatched - language: system - types: [json] - require_serial: true - id: rustfmt name: rustfmt entry: cargo fmt -- diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 4403c47..3891501 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -62,7 +62,7 @@ blocks: jobs: - name: "Build Opsqueue binary (in release mode) & run unit tests" commands: - - "just nix-build-bin | cachix push channable" + - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build-bin)) | grep -v '\\.drv$' | cachix push channable" # Once a day, we update the devenv cache. # Except when fresh, this takes < 1 second # but it cannot live in the prologue @@ -70,7 +70,7 @@ blocks: - cache store nix-store-$(date -u -Idate) /nix - name: "Build Python library (in release mode)" commands: - - "just nix-build-python | cachix push channable" + - "nix-store -qR --include-outputs $(nix-store -qd $(just nix-build-python)) | grep -v '\\.drv$' | cachix push channable" - name: "Integration" dependencies: - "Build" diff --git a/biome.json b/biome.json deleted file mode 100644 index 8c938a8..0000000 --- a/biome.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "formatter": { - "enabled": true, - "formatWithErrors": true, - "indentStyle": "space", - "lineWidth": 100 - }, - "vcs": { - "enabled": true, - "clientKind": "git", - "useIgnoreFile": true - }, - "files": { - "ignore": ["nix/**"], - "maxSize": 20971520 - } -} diff --git a/default.nix b/default.nix index ca28c9e..e84fd2a 100644 --- a/default.nix +++ b/default.nix @@ -4,7 +4,7 @@ let pkgs = import ./nix/nixpkgs-pinned.nix { }; - pythonEnv = pkgs.pythonChannable.withPackages ( + pythonEnv = pkgs.python.withPackages ( p: with p; [ click mypy @@ -22,9 +22,6 @@ let ] ); - # We choose a minimal Rust channel to keep the Nix closure size smaller - rust = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; - defaultEnv = pkgs.buildEnv { name = "opsqueue-env-default"; paths = [ @@ -39,7 +36,7 @@ let pkgs.ruff # For compiling the Rust parts - rust + pkgs.rustToolchain pkgs.sqlx-cli # Manage nix pins diff --git a/justfile b/justfile index bf16498..b845ef9 100644 --- a/justfile +++ b/justfile @@ -47,21 +47,21 @@ test-integration *TEST_ARGS: build-bin build-python cd libs/opsqueue_python source "./.setup_local_venv.sh" - timeout 30 pytest --color=yes {{TEST_ARGS}} + pytest --color=yes {{TEST_ARGS}} # Python integration test suite, using artefacts built through Nix. Args are forwarded to pytest [group('nix')] -nix-test-integration *TEST_ARGS: +nix-test-integration *TEST_ARGS: nix-build-bin #!/usr/bin/env bash set -euxo pipefail nix_build_python_library_dir=$(just nix-build-python) cd libs/opsqueue_python/tests - export PYTHONPATH="${nix_build_python_library_dir}/lib/python3.12/site-packages" + export PYTHONPATH="${nix_build_python_library_dir}/lib/python3.13/site-packages" export OPSQUEUE_VIA_NIX=true export RUST_LOG="opsqueue=debug" - timeout 30 pytest --color=yes {{TEST_ARGS}} + pytest --color=yes {{TEST_ARGS}} # Run all linters, fast and slow [group('lint')] @@ -89,7 +89,7 @@ mypy: # Build Nix-derivations of binary and all libraries (release profile) [group('nix')] -nix-build: (_nix-build "opsqueue" "pythonChannable.pkgs.opsqueue_python") +nix-build: (_nix-build "opsqueue" "python.pkgs.opsqueue_python") # Build Nix-derivation of binary (release profile) [group('nix')] @@ -97,7 +97,7 @@ nix-build-bin: (_nix-build "opsqueue") # Build Nix-derivation of Python client library (release profile) [group('nix')] -nix-build-python: (_nix-build "pythonChannable.pkgs.opsqueue_python") +nix-build-python: (_nix-build "python.pkgs.opsqueue_python") _nix-build +TARGETS: nix build --file nix/nixpkgs-pinned.nix --print-out-paths --print-build-logs --no-link --option sandbox true {{TARGETS}} diff --git a/libs/opsqueue_python/examples/tracing/tracing_consumer.py b/libs/opsqueue_python/examples/tracing/tracing_consumer.py index 96e16e7..a319117 100644 --- a/libs/opsqueue_python/examples/tracing/tracing_consumer.py +++ b/libs/opsqueue_python/examples/tracing/tracing_consumer.py @@ -8,7 +8,7 @@ # ConsoleSpanExporter, ) from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.resources import SERVICE_NAME, Resource # type: ignore[attr-defined] logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) diff --git a/libs/opsqueue_python/examples/tracing/tracing_producer.py b/libs/opsqueue_python/examples/tracing/tracing_producer.py index 2d9a1ed..24f0f13 100644 --- a/libs/opsqueue_python/examples/tracing/tracing_producer.py +++ b/libs/opsqueue_python/examples/tracing/tracing_producer.py @@ -13,7 +13,7 @@ # ConsoleSpanExporter, ) from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.resources import SERVICE_NAME, Resource # type: ignore[attr-defined] from opsqueue.producer import ProducerClient @@ -75,7 +75,7 @@ def added_baggage( yield finally: for attached_token in attached_context_tokens: - opentelemetry.context.detach(attached_token) + opentelemetry.context.detach(attached_token) # type: ignore[arg-type] if __name__ == "__main__": diff --git a/libs/opsqueue_python/opsqueue_python.nix b/libs/opsqueue_python/opsqueue_python.nix index d392480..e963017 100644 --- a/libs/opsqueue_python/opsqueue_python.nix +++ b/libs/opsqueue_python/opsqueue_python.nix @@ -1,61 +1,120 @@ { + # Builtin + pkgs, lib, + + # Rust version. (Override this with an overlay if you like) + rustToolchain, + + # Native build dependencies: + maturin, buildPythonPackage, - rustPlatform, perl, git, - # Python packages: + + # Python package dependencies: cbor2, opentelemetry-api, opentelemetry-exporter-otlp, opentelemetry-sdk, }: let - root = ../../.; - util = import (root + /nix/util.nix) { inherit lib; }; -in -buildPythonPackage rec { - pname = "opsqueue"; - version = "0.1.0"; - pyproject = true; - - src = util.fileFilter { - name = "opsqueue_python"; - src = root; + python = pkgs.python3; + sources = import ../../nix/sources.nix; + crane = import sources.crane { pkgs = pkgs; }; + craneLib = crane.overrideToolchain (pkgs: rustToolchain); - # We're copying slightly too much to the Nix store here, - # but using the more granular file filter was very error-prone. - # This is one thing that could be improved a little in the future. - srcGlobalWhitelist = [ - ".py" - ".pyi" - "py.typed" - ".rs" - ".toml" - ".lock" - ".db" - ".md" - ]; + # Only the files necessary to build the Rust-side and cache dependencies + sqlFileFilter = path: _type: builtins.match "^.*\.(db|sql)$" path != null; + rustCrateFileFilter = + path: type: (sqlFileFilter path type) || (craneLib.filterCargoSources path type); + depsSrc = lib.cleanSourceWith { + src = ../../.; + name = "opsqueue"; + filter = rustCrateFileFilter; }; - cargoDeps = rustPlatform.importCargoLock { lockFile = ../../Cargo.lock; }; + # Full set of files to build the Python wheel on top + pythonFileFilter = path: _type: builtins.match "^.*\.(py|md)$" path != null; + wheelFileFilter = path: type: (pythonFileFilter path type) || (rustCrateFileFilter path type); + wheelSrc = lib.cleanSourceWith { + src = ../../.; + name = "opsqueue"; + filter = wheelFileFilter; + }; - env = { - DATABASE_URL = "sqlite://./opsqueue/opsqueue_example_database_schema.db"; + crateName = craneLib.crateNameFromCargoToml { cargoToml = ./Cargo.toml; }; + pname = crateName.pname; + version = crateName.version; + commonArgs = { + inherit version pname; + src = depsSrc; + strictDeps = true; + nativeBuildInputs = [ python ]; + cargoExtraArgs = "--package opsqueue_python"; + doCheck = false; }; + cargoArtifacts = craneLib.buildDepsOnly (commonArgs // { pname = pname; }); - pythonImportsCheck = [ pname ]; + wheelTail = "py3-abi3-linux_x86_64"; + wheelName = "opsqueue-${version}-${wheelTail}.whl"; - maturinBuildFlags = [ - "--manifest-path" - "./libs/opsqueue_python/Cargo.toml" - ]; + crateWheel = + (craneLib.buildPackage ( + commonArgs + // { + inherit cargoArtifacts; + src = wheelSrc; + } + )).overrideAttrs + (old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ maturin ]; + env.PYO3_PYTHON = python.interpreter; + + # We intentionally _override_ rather than extend the buildPhase + # as Maturin itself calls `cargo build`, no need to call it twice. + buildPhase = '' + cargo --version + maturin build --release --offline --target-dir ./target --manifest-path "./libs/opsqueue_python/Cargo.toml" + ''; - nativeBuildInputs = with rustPlatform; [ - perl - git - cargoSetupHook - maturinBuildHook + # We build a single wheel + # but by convention its name is based on the precise combination of + # Python version + OS version + architecture + ... + # + # The Nix hash already covers us for uniqueness and compatibility. + # So this 'trick' copies it to a predictably named file. + # + # Just like `buildPhase`, we override rather than extend + # because we are only interested in the wheel output of Maturin as a whole. + # (which is an archive inside of it containing the `.so` cargo built) + installPhase = '' + mkdir -p $out + for wheel in ./target/wheels/*.whl ; do + cp "$wheel" $out/${wheelName} + done + ''; + + # There are no Rust unit tests in the Python FFI library currently, + # so we can skip rebuilding opsqueue_python for tests. + doCheck = false; + }); +in +buildPythonPackage { + pname = pname; + format = "wheel"; + version = crateName.version; + src = "${crateWheel}/${wheelName}"; + doCheck = false; + pythonImportsCheck = [ + "opsqueue" + # Internal: This is the most important one! + "opsqueue.opsqueue_internal" + # Visible: + "opsqueue.producer" + "opsqueue.consumer" + "opsqueue.exceptions" + "opsqueue.common" ]; propagatedBuildInputs = [ diff --git a/libs/opsqueue_python/pyproject.toml b/libs/opsqueue_python/pyproject.toml index 097f0ee..21be1d1 100644 --- a/libs/opsqueue_python/pyproject.toml +++ b/libs/opsqueue_python/pyproject.toml @@ -52,15 +52,15 @@ test = [ "pytest==8.3.3", "pytest-random-order==1.1.1", "pytest-parallel==0.1.1", - "pytest-timeout==2.4.0", + # "pytest-timeout==2.4.0", "py==1.11.0", # Needs to be manually specified because of this issue: https://github.com/kevlened/pytest-parallel/issues/118 ] [tool.pytest.ini_options] # We ensure tests never rely on global state, # by running them in a random order, and in parallel: -addopts = "--random-order --workers=auto" -# Individual tests should be very fast. They should never take multiple seconds -# If after 20sec (accomodating for a toaster-like PC) there is no progress, -# assume a deadlock -timeout=20 +addopts = "--random-order --workers=4" +# # Individual tests should be very fast. They should never take multiple seconds +# # If after 20sec (accomodating for a toaster-like PC) there is no progress, +# # assume a deadlock +# timeout=20 diff --git a/libs/opsqueue_python/src/common.rs b/libs/opsqueue_python/src/common.rs index 91d13a4..ba5e8aa 100644 --- a/libs/opsqueue_python/src/common.rs +++ b/libs/opsqueue_python/src/common.rs @@ -428,11 +428,8 @@ pub async fn check_signals_in_background() -> FatalPythonException { if let Err(err) = py.check_signals() { // A signal was triggered Some(err) - } else if let Some(err) = PyErr::take(py) { - // A non-signal Python exception was thrown - return Some(err); } else { - return None; + PyErr::take(py) } }); if let Some(res) = res { diff --git a/libs/opsqueue_python/src/consumer.rs b/libs/opsqueue_python/src/consumer.rs index 1870ba2..613ac5c 100644 --- a/libs/opsqueue_python/src/consumer.rs +++ b/libs/opsqueue_python/src/consumer.rs @@ -418,7 +418,7 @@ impl ConsumerClient { } done_count = done_count.saturating_add(1); - if done_count % 50 == 0 { + if done_count.is_multiple_of(50) { tracing::info!("Processed {} chunks", done_count); } } diff --git a/nix/nixpkgs-pinned.nix b/nix/nixpkgs-pinned.nix index 8378efa..6e3d386 100644 --- a/nix/nixpkgs-pinned.nix +++ b/nix/nixpkgs-pinned.nix @@ -10,7 +10,8 @@ let used_overlays = [ (import sources.rust-overlay) (import ./overlay.nix) - ] ++ overlays; + ] + ++ overlays; nixpkgsArgs = args // { overlays = used_overlays; diff --git a/nix/overlay.nix b/nix/overlay.nix index 3f8c455..f5ff633 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -1,16 +1,26 @@ # Overlay for Nixpkgs which holds all opsqueue related packages. final: prev: let + sources = import ./sources.nix; pythonOverlay = import ./python-overlay.nix; + + # We want to use the same Rust version in Nix + # as we use when _not_ using Nix. + # + # When using opsqueue as part of your own Nix derivations, + # be sure to use an overlay to set `rustToolchain` + # to the desired Rust version + # if you want to use a different version from the default. + rustToolchain = final.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml; + + crane = import sources.crane { pkgs = final; }; + craneLib = crane.overrideToolchain (pkgs: rustToolchain); + python3 = final.python313.override { packageOverrides = pythonOverlay; }; in { - opsqueue = final.callPackage ../opsqueue/opsqueue.nix { }; + inherit rustToolchain python3; + python = python3; - # The explicit choice is made not to override `python312`, as this will cause a rebuild of many - # packages when nixpkgs uses python 3.12 as default python environment. - # These packages should not be affected, e.g. cachix. This is because of a transitive - # dependency on the Python packages that we override. - # In our case cachix > ghc > shpinx > Python libraries. - pythonChannable = prev.python312.override { packageOverrides = pythonOverlay; }; + opsqueue = final.callPackage ../opsqueue/opsqueue.nix { }; } diff --git a/nix/sources.json b/nix/sources.json index 3614497..42a6945 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -1,14 +1,26 @@ { + "crane": { + "branch": "master", + "description": "A Nix library for building cargo projects. Never build twice thanks to incremental artifact caching.", + "homepage": "https://crane.dev", + "owner": "ipetkov", + "repo": "crane", + "rev": "0bda7e7d005ccb5522a76d11ccfbf562b71953ca", + "sha256": "1ndhiw867qiwr3kswm2mb81y3xcfdwz92xvs00r6jd4blxsv7z09", + "type": "tarball", + "url": "https://github.com/ipetkov/crane/archive/0bda7e7d005ccb5522a76d11ccfbf562b71953ca.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "nixpkgs": { "branch": "nixpkgs-unstable", "description": "Nix Packages collection", "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d19cf9dfc633816a437204555afeb9e722386b76", - "sha256": "1wirhlw7cqaypgakyfz9ikv7nxdq3il0fk38cdrdmps2zn1l4ccp", + "rev": "6308c3b21396534d8aaeac46179c14c439a89b8a", + "sha256": "14qnx22pkl9v4r0lxnnz18f4ybxj8cv18hyf1klzap98hckg58y4", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/d19cf9dfc633816a437204555afeb9e722386b76.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/6308c3b21396534d8aaeac46179c14c439a89b8a.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "rust-overlay": { diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index 14b0112..272214d 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -5,11 +5,11 @@ edition = "2021" description = "lightweight batch processing queue for heavy loads" repository = "https://github.com/channable/opsqueue" license = "MIT" +include=["opsqueue_example_database_schema.db"] [lib] name="opsqueue" path="src/lib.rs" -include=["opsqueue_example_database_schema.db"] [[bin]] name="opsqueue" diff --git a/opsqueue/opsqueue.nix b/opsqueue/opsqueue.nix index 267907a..76cb866 100644 --- a/opsqueue/opsqueue.nix +++ b/opsqueue/opsqueue.nix @@ -1,72 +1,44 @@ { + pkgs, lib, - rustPlatform, - # Building options - buildType ? "release", - # Testing options - checkType ? "debug", - doCheck ? true, - useNextest ? false, # Disabled for now. Re-enable as part of https://github.com/channable/opsqueue/issues/81 - perl, + rustToolchain, git, }: let - root = ../.; - util = import (root + /nix/util.nix) { inherit lib; }; -in -rustPlatform.buildRustPackage { - name = "opsqueue"; - inherit - buildType - checkType - doCheck - useNextest - ; - - src = util.fileFilter { + sources = import ../nix/sources.nix; + crane = import sources.crane { pkgs = pkgs; }; + craneLib = crane.overrideToolchain (pkgs: rustToolchain); + extraFileFilter = path: _type: builtins.match "^.*\.(db|sql)$" path != null; + fileFilter = path: type: (extraFileFilter path type) || (craneLib.filterCargoSources path type); + + # src = craneLib.cleanCargoSource ../.; + src = lib.cleanSourceWith { + src = ../.; name = "opsqueue"; - src = ./.; - - srcWhitelist = [ - "Cargo.toml" - ".cargo(/.*)?" - "build\.rs" - "opsqueue_example_database_schema\.db" - "app(/.*)?" - "migrations(/.*)?" - "src(/.*)?" - ]; - - srcGlobalWhitelist = [ - ".lock" - ".toml" - ".rs" - ".db" - ".sql" - ]; - }; - - postUnpack = '' - cp "${../Cargo.lock}" "/build/opsqueue/Cargo.lock" - chmod +w /build/opsqueue/Cargo.lock - ''; - - env = { - DATABASE_URL = "sqlite:///build/opsqueue/opsqueue_example_database_schema.db"; + filter = fileFilter; }; - nativeBuildInputs = [ - perl - git - ]; - - cargoLock = { - lockFile = ../Cargo.lock; + crateName = craneLib.crateNameFromCargoToml { cargoToml = ../opsqueue/Cargo.toml; }; + pname = crateName.pname; + version = crateName.version; + commonArgs = { + inherit src version pname; + strictDeps = true; + nativeBuildInputs = [ ]; + cargoExtraArgs = "--package opsqueue"; + doCheck = true; }; - - # This limits the build to only build the opsqueue executable - cargoBuildFlags = [ - "--package" - "opsqueue" - ]; -} + cargoArtifacts = craneLib.buildDepsOnly commonArgs; +in +craneLib.buildPackage ( + commonArgs + // { + inherit cargoArtifacts; + + # Needed for the SQLx macros: + env = { + DATABASE_URL = "sqlite:///build/opsqueue/opsqueue/opsqueue_example_database_schema.db"; + }; + + } +) diff --git a/opsqueue/src/consumer/server/mod.rs b/opsqueue/src/consumer/server/mod.rs index c869f5d..412391e 100644 --- a/opsqueue/src/consumer/server/mod.rs +++ b/opsqueue/src/consumer/server/mod.rs @@ -200,7 +200,7 @@ impl Completer { } // Log some indication of progress every so often: self.count = self.count.saturating_add(1); - if self.count % 1000 == 0 { + if self.count.is_multiple_of(1000) { tracing::info!("Processed {} chunks", self.count); } }