tezlm/conduit
1
0
Fork 0
Code Issues Pull requests Releases Activity

Merge branch 'always-cache-attic' into 'next'

Browse source 
ci: prevent unnecessary rebuilds

See merge request famedly/conduit!662
This commit is contained in:
Matthias Ahouansou 2024-05-05 05:54:16 +00:00
commit bed9072a69
7 changed files with 392 additions and 279 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitlab-ci.yml
View file

@ -45,13 +45,13 @@ before_script:
# Set CARGO_HOME to a cacheable path
- export CARGO_HOME="$(git rev-parse --show-toplevel)/.gitlab-ci.d/cargo"
# Cache attic client
- ./bin/nix-build-and-cache --inputs-from . attic
ci:
stage: ci
image: nixos/nix:2.20.4
script:
# Cache attic client
- ./bin/nix-build-and-cache --inputs-from . attic
# Cache the inputs required for the devShell
- ./bin/nix-build-and-cache .#devShells.x86_64-linux.default.inputDerivation

355
flake.nix
View file

@ -19,294 +19,97 @@
attic.url = "github:zhaofengli/attic?ref=main";
};
outputs =
{ self
, nixpkgs
, flake-utils
, nix-filter
, fenix
, crane
, ...
}: flake-utils.lib.eachDefaultSystem (system:
outputs = inputs:
let
pkgsHost = nixpkgs.legacyPackages.${system};
# Keep sorted
mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: {
craneLib =
(inputs.crane.mkLib pkgs).overrideToolchain self.toolchain;
# Nix-accessible `Cargo.toml`
cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml);
default = self.callPackage ./nix/pkgs/default {};
# The Rust toolchain to use
toolchain = fenix.packages.${system}.fromToolchainFile {
file = ./rust-toolchain.toml;
inherit inputs;
# See also `rust-toolchain.toml`
sha256 = "sha256-SXRtAuO4IqNOQq+nLbrsDFbVk+3aVA8NNpSZsKlVH/8=";
};
oci-image = self.callPackage ./nix/pkgs/oci-image {};
builder = pkgs:
((crane.mkLib pkgs).overrideToolchain toolchain).buildPackage;
book = self.callPackage ./nix/pkgs/book {};
nativeBuildInputs = pkgs: [
# bindgen needs the build platform's libclang. Apparently due to
# "splicing weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't
# quite do the right thing here.
pkgs.pkgsBuildHost.rustPlatform.bindgenHook
];
rocksdb =
let
version = "9.1.1";
in
pkgs.rocksdb.overrideAttrs (old: {
inherit version;
src = pkgs.fetchFromGitHub {
owner = "facebook";
repo = "rocksdb";
rev = "v${version}";
hash = "sha256-/Xf0bzNJPclH9IP80QNaABfhj4IAR5LycYET18VFCXc=";
};
});
rocksdb' = pkgs:
let
version = "9.1.1";
in
pkgs.rocksdb.overrideAttrs (old: {
inherit version;
src = pkgs.fetchFromGitHub {
owner = "facebook";
repo = "rocksdb";
rev = "v${version}";
hash = "sha256-/Xf0bzNJPclH9IP80QNaABfhj4IAR5LycYET18VFCXc=";
};
shell = self.callPackage ./nix/shell.nix {};
# The Rust toolchain to use
toolchain = inputs
.fenix
.packages
.${pkgs.pkgsBuildHost.system}
.fromToolchainFile {
file = ./rust-toolchain.toml;
# See also `rust-toolchain.toml`
sha256 = "sha256-SXRtAuO4IqNOQq+nLbrsDFbVk+3aVA8NNpSZsKlVH/8=";
};
});
in
inputs.flake-utils.lib.eachDefaultSystem (system:
let
pkgs = inputs.nixpkgs.legacyPackages.${system};
in
{
packages = {
default = (mkScope pkgs).default;
oci-image = (mkScope pkgs).oci-image;
book = (mkScope pkgs).book;
}
//
builtins.listToAttrs
(builtins.concatLists
(builtins.map
(crossSystem:
let
binaryName = "static-${crossSystem}";
pkgsCrossStatic =
(import inputs.nixpkgs {
inherit system;
crossSystem = {
config = crossSystem;
};
}).pkgsStatic;
in
[
# An output for a statically-linked binary
{
name = binaryName;
value = (mkScope pkgsCrossStatic).default;
}
env = pkgs: {
CONDUIT_VERSION_EXTRA = self.shortRev or self.dirtyShortRev;
ROCKSDB_INCLUDE_DIR = "${rocksdb' pkgs}/include";
ROCKSDB_LIB_DIR = "${rocksdb' pkgs}/lib";
}
// pkgs.lib.optionalAttrs pkgs.stdenv.hostPlatform.isStatic {
ROCKSDB_STATIC = "";
}
// {
CARGO_BUILD_RUSTFLAGS = let inherit (pkgs) lib stdenv; in
lib.concatStringsSep " " ([]
++ lib.optionals
# This disables PIE for static builds, which isn't great in terms
# of security. Unfortunately, my hand is forced because nixpkgs'
# `libstdc++.a` is built without `-fPIE`, which precludes us from
# leaving PIE enabled.
stdenv.hostPlatform.isStatic
["-C" "relocation-model=static"]
++ lib.optionals
(stdenv.buildPlatform.config != stdenv.hostPlatform.config)
["-l" "c"]
++ lib.optionals
# This check has to match the one [here][0]. We only need to set
# these flags when using a different linker. Don't ask me why,
# though, because I don't know. All I know is it breaks otherwise.
#
# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40
(
# Nixpkgs doesn't check for x86_64 here but we do, because I
# observed a failure building statically for x86_64 without
# including it here. Linkers are weird.
(stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64)
&& stdenv.hostPlatform.isStatic
&& !stdenv.isDarwin
&& !stdenv.cc.bintools.isLLVM
# An output for an OCI image based on that binary
{
name = "oci-image-${crossSystem}";
value = (mkScope pkgsCrossStatic).oci-image;
}
]
)
[
"-l"
"stdc++"
"-L"
"${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib"
]
);
}
# What follows is stolen from [here][0]. Its purpose is to properly
# configure compilers and linkers for various stages of the build, and
# even covers the case of build scripts that need native code compiled and
# run on the build platform (I think).
#
# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80
// (
let
inherit (pkgs.rust.lib) envVars;
in
pkgs.lib.optionalAttrs
(pkgs.stdenv.targetPlatform.rust.rustcTarget
!= pkgs.stdenv.hostPlatform.rust.rustcTarget)
(
let
inherit (pkgs.stdenv.targetPlatform.rust) cargoEnvVarTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForTarget;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" =
envVars.linkerForTarget;
}
)
// (
let
inherit (pkgs.stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForHost;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForHost;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost;
CARGO_BUILD_TARGET = rustcTarget;
}
)
// (
let
inherit (pkgs.stdenv.buildPlatform.rust) cargoEnvVarTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
HOST_CC = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/cc";
HOST_CXX = "${pkgs.pkgsBuildHost.stdenv.cc}/bin/c++";
}
));
package = pkgs: builder pkgs {
src = nix-filter {
root = ./.;
include = [
"src"
"Cargo.toml"
"Cargo.lock"
];
};
# This is redundant with CI
doCheck = false;
env = env pkgs;
nativeBuildInputs = nativeBuildInputs pkgs;
meta.mainProgram = cargoToml.package.name;
};
mkOciImage = pkgs: package:
pkgs.dockerTools.buildImage {
name = package.pname;
tag = "next";
copyToRoot = [
pkgs.dockerTools.caCertificates
];
config = {
# Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT)
# are handled as expected
Entrypoint = [
"${pkgs.lib.getExe' pkgs.tini "tini"}"
"--"
];
Cmd = [
"${pkgs.lib.getExe package}"
];
};
};
in
{
packages = {
default = package pkgsHost;
oci-image = mkOciImage pkgsHost self.packages.${system}.default;
book =
let
package = self.packages.${system}.default;
in
pkgsHost.stdenv.mkDerivation {
pname = "${package.pname}-book";
version = package.version;
src = nix-filter {
root = ./.;
include = [
"book.toml"
"conduit-example.toml"
"README.md"
"debian/README.md"
"docs"
];
};
nativeBuildInputs = (with pkgsHost; [
mdbook
]);
buildPhase = ''
mdbook build
mv public $out
'';
};
}
//
builtins.listToAttrs
(builtins.concatLists
(builtins.map
(crossSystem:
let
binaryName = "static-${crossSystem}";
pkgsCrossStatic =
(import nixpkgs {
inherit system;
crossSystem = {
config = crossSystem;
};
}).pkgsStatic;
in
[
# An output for a statically-linked binary
{
name = binaryName;
value = package pkgsCrossStatic;
}
# An output for an OCI image based on that binary
{
name = "oci-image-${crossSystem}";
value = mkOciImage
pkgsCrossStatic
self.packages.${system}.${binaryName};
}
"x86_64-unknown-linux-musl"
"aarch64-unknown-linux-musl"
]
)
[
"x86_64-unknown-linux-musl"
"aarch64-unknown-linux-musl"
]
)
);
);
devShells.default = pkgsHost.mkShell {
env = env pkgsHost // {
# Rust Analyzer needs to be able to find the path to default crate
# sources, and it can read this environment variable to do so. The
# `rust-src` component is required in order for this to work.
RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library";
};
# Development tools
nativeBuildInputs = nativeBuildInputs pkgsHost ++ [
# Always use nightly rustfmt because most of its options are unstable
#
# This needs to come before `toolchain` in this list, otherwise
# `$PATH` will have stable rustfmt instead.
fenix.packages.${system}.latest.rustfmt
toolchain
] ++ (with pkgsHost; [
engage
# Needed for producing Debian packages
cargo-deb
# Needed for Complement
go
olm
# Needed for our script for Complement
jq
# Needed for finding broken markdown links
lychee
# Useful for editing the book locally
mdbook
]);
};
});
devShells.default = (mkScope pkgs).shell;
}
);
}

34
nix/pkgs/book/default.nix Normal file
View file

@ -0,0 +1,34 @@
# Keep sorted
{ default
, inputs
, mdbook
, stdenv
}:
stdenv.mkDerivation {
pname = "${default.pname}-book";
version = default.version;
src = let filter = inputs.nix-filter.lib; in filter {
root = inputs.self;
# Keep sorted
include = [
"book.toml"
"conduit-example.toml"
"debian/README.md"
"docs"
"README.md"
];
};
nativeBuildInputs = [
mdbook
];
buildPhase = ''
mdbook build
mv public $out
'';
}

100
nix/pkgs/default/cross-compilation-env.nix Normal file
View file

@ -0,0 +1,100 @@
{ lib
, pkgsBuildHost
, rust
, stdenv
}:
lib.optionalAttrs stdenv.hostPlatform.isStatic {
ROCKSDB_STATIC = "";
}
//
{
CARGO_BUILD_RUSTFLAGS =
lib.concatStringsSep
" "
([]
# This disables PIE for static builds, which isn't great in terms of
# security. Unfortunately, my hand is forced because nixpkgs'
# `libstdc++.a` is built without `-fPIE`, which precludes us from
# leaving PIE enabled.
++ lib.optionals
stdenv.hostPlatform.isStatic
[ "-C" "relocation-model=static" ]
++ lib.optionals
(stdenv.buildPlatform.config != stdenv.hostPlatform.config)
[ "-l" "c" ]
++ lib.optionals
# This check has to match the one [here][0]. We only need to set
# these flags when using a different linker. Don't ask me why, though,
# because I don't know. All I know is it breaks otherwise.
#
# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40
(
# Nixpkgs doesn't check for x86_64 here but we do, because I
# observed a failure building statically for x86_64 without
# including it here. Linkers are weird.
(stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64)
&& stdenv.hostPlatform.isStatic
&& !stdenv.isDarwin
&& !stdenv.cc.bintools.isLLVM
)
[
"-l"
"stdc++"
"-L"
"${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib"
]
);
}
# What follows is stolen from [here][0]. Its purpose is to properly configure
# compilers and linkers for various stages of the build, and even covers the
# case of build scripts that need native code compiled and run on the build
# platform (I think).
#
# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80
//
(
let
inherit (rust.lib) envVars;
in
lib.optionalAttrs
(stdenv.targetPlatform.rust.rustcTarget
!= stdenv.hostPlatform.rust.rustcTarget)
(
let
inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForTarget;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" =
envVars.linkerForTarget;
}
)
//
(
let
inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForHost;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForHost;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost;
CARGO_BUILD_TARGET = rustcTarget;
}
)
//
(
let
inherit (stdenv.buildPlatform.rust) cargoEnvVarTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild;
HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc";
HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++";
}
)
)

96
nix/pkgs/default/default.nix Normal file
View file

@ -0,0 +1,96 @@
# Dependencies (keep sorted)
{ craneLib
, inputs
, lib
, pkgsBuildHost
, rocksdb
, rust
, stdenv
# Options (keep sorted)
, default-features ? true
, features ? []
, profile ? "release"
}:
let
buildDepsOnlyEnv =
let
rocksdb' = rocksdb.override {
enableJemalloc = builtins.elem "jemalloc" features;
};
in
{
NIX_OUTPATH_USED_AS_RANDOM_SEED = "randomseed"; # https://crane.dev/faq/rebuilds-bindgen.html
ROCKSDB_INCLUDE_DIR = "${rocksdb'}/include";
ROCKSDB_LIB_DIR = "${rocksdb'}/lib";
}
//
(import ./cross-compilation-env.nix {
# Keep sorted
inherit
lib
pkgsBuildHost
rust
stdenv;
});
buildPackageEnv = {
# Temporarily disabled, see https://gitlab.com/famedly/conduit/-/merge_requests/662#note_1892753424
# CONDUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev;
} // buildDepsOnlyEnv;
commonAttrs = {
inherit
(craneLib.crateNameFromCargoToml {
cargoToml = "${inputs.self}/Cargo.toml";
})
pname
version;
src = let filter = inputs.nix-filter.lib; in filter {
root = inputs.self;
# Keep sorted
include = [
"Cargo.lock"
"Cargo.toml"
"src"
];
};
nativeBuildInputs = [
# bindgen needs the build platform's libclang. Apparently due to "splicing
# weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the
# right thing here.
pkgsBuildHost.rustPlatform.bindgenHook
];
CARGO_PROFILE = profile;
};
in
craneLib.buildPackage ( commonAttrs // {
cargoArtifacts = craneLib.buildDepsOnly (commonAttrs // {
env = buildDepsOnlyEnv;
});
cargoExtraArgs = "--locked "
+ lib.optionalString
(!default-features)
"--no-default-features "
+ lib.optionalString
(features != [])
"--features " + (builtins.concatStringsSep "," features);
# This is redundant with CI
doCheck = false;
env = buildPackageEnv;
passthru = {
env = buildPackageEnv;
};
meta.mainProgram = commonAttrs.pname;
})

25
nix/pkgs/oci-image/default.nix Normal file
View file

@ -0,0 +1,25 @@
# Keep sorted
{ default
, dockerTools
, lib
, tini
}:
dockerTools.buildImage {
name = default.pname;
tag = "next";
copyToRoot = [
dockerTools.caCertificates
];
config = {
# Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT)
# are handled as expected
Entrypoint = [
"${lib.getExe' tini "tini"}"
"--"
];
Cmd = [
"${lib.getExe default}"
];
};
}

55
nix/shell.nix Normal file
View file

@ -0,0 +1,55 @@
# Keep sorted
{ cargo-deb
, default
, engage
, go
, inputs
, jq
, lychee
, mdbook
, mkShell
, olm
, system
, toolchain
}:
mkShell {
env = default.env // {
# Rust Analyzer needs to be able to find the path to default crate
# sources, and it can read this environment variable to do so. The
# `rust-src` component is required in order for this to work.
RUST_SRC_PATH = "${toolchain}/lib/rustlib/src/rust/library";
};
# Development tools
nativeBuildInputs = default.nativeBuildInputs ++ [
# Always use nightly rustfmt because most of its options are unstable
#
# This needs to come before `toolchain` in this list, otherwise
# `$PATH` will have stable rustfmt instead.
inputs.fenix.packages.${system}.latest.rustfmt
# Keep sorted
engage
toolchain
# Needed for producing Debian packages
cargo-deb
# Needed for our script for Complement
jq
# Needed for Complement
go
olm
# Needed for our script for Complement
jq
# Needed for finding broken markdown links
lychee
# Useful for editing the book locally
mdbook
];
}