divnix
Straightforward example implementing operables/containers?
my apologies if I missed it, but it would be nice if there was some code linked in the readme like std-book that also implemented operables/containers for newbs like me that you could just copy&paste and iterate on
this video had some good examples but I'm not sure where the code they used in the video is store https://www.loom.com/share/27d91aa1eac24bcaaaed18ea6d6d03ca
found this via https://github.com/search?q=%22divnix%2Fstd%22+path%3Aflake.nix+operables&type=code
still not tons of examples for that search
(e.g. I am just trying to run postgresql in a container in a "std" way)
I think after landing https://github.com/divnix/std/pull/297, I'd be at least able to informally share some more, including arion integration. I'll probably repost a snippet here, for a "quick-fix", well aware that this ticket rightfully asks for a larger body of examples.
Ok, so let me share this example:
❯ l nix/cardano-stack # the Cardano Stack Software System / Cell
Permissions Size User Date Modified Name
drwxrwxr-x - blaggacao 17 May 17:35 deployments
.rw-rw-r-- 4.1k blaggacao 29 May 12:37 deployments.nix
.rw-rw-r-- 2.3k blaggacao 29 May 12:37 entrypoints.nix
.rw-rw-r-- 585 blaggacao 29 May 12:37 oci-images.nix
.rw-rw-r-- 108 blaggacao 31 May 12:54 packages.nix
.rw-rw-r-- 1.5k blaggacao 29 May 12:37 testbed.nix
# ./packages.nix
# in this case, packages are packaged in upstream flake
{
inherit
(inputs.offchain-metadata-tools.app.packages)
metadata-server
metadata-sync
;
}
# ./entrypoints.nix # could also call them `operables.nix`
# tries to capture the runtime contract explicitly in a format
# that is easy to communicate (bash)
let
inherit (inputs) std nixpkgs;
dbConnectionEnv = ''
if [[ -z "''${DB_NAME:-}" ]]; then
echo DB_NAME must be explicitly set
exit 1
fi
echo "Metadata will be stored in the ''${DB_NAME} database..."
if [[ -z "''${DB_USER:-}" ]]; then
echo DB_USER must be explicitly set
exit 1
fi
echo "Metadata will be accessed via the ''${DB_USER} database user..."
if [[ -z "''${DB_PASS:-}" ]]; then
echo DB_PASS must be explicitly set
exit 1
fi
if [[ -z "''${DB_HOST:-}" ]]; then
echo DB_HOST must be explicitly set
exit 1
fi
echo "Metadata will be accessed via the ''${DB_HOST} database host..."
if [[ -z "''${DB_PORT:-}" ]]; then
echo DB_PORT must be explicitly set
exit 1
fi
echo "Metadata will be accessed via the ''${DB_PORT} database host..."
'';
in {
metadata-server = std.lib.ops.mkOperable {
package = cell.packages.metadata-server;
runtimeScript = ''
${dbConnectionEnv}
if [[ -z "''${PORT:-}" ]]; then
echo PORT must be explicitly set
exit 1
fi
echo "Metadata will be served on port ''${PORT}..."
exec ${cell.packages.metadata-server}/bin/metadata-server \
--db "$DB_NAME" \
--db-user "$DB_USER" \
--db-pass "$DB_PASS" \
--db-host "$DB_HOST" \
--db-port "$DB_PORT" \
--port "$PORT"
'';
};
metadata-sync = std.lib.ops.mkOperable {
package = cell.packages.metadata-sync;
runtimeInputs = [nixpkgs.gitMinimal];
runtimeScript = ''
${dbConnectionEnv}
if [[ -z "''${GIT_URL:-}" ]]; then
echo GIT_URL must be explicitly set
exit 1
fi
echo "Metadata will be sync from ''${GIT_URL}..."
if [[ -z "''${GIT_METADATA_FOLDER:-}" ]]; then
echo GIT_METADATA_FOLDER must be explicitly set
exit 1
fi
echo "Metadata will be sync from the folder ''${GIT_METADATA_FOLDER}..."
exec ${cell.packages.metadata-sync}/bin/metadata-sync \
--db "$DB_NAME" \
--db-user "$DB_USER" \
--db-pass "$DB_PASS" \
--db-host "$DB_HOST" \
--db-port "$DB_PORT" \
--git-url "$GIT_URL" \
--git-metadata-folder "$GIT_METADATA_FOLDER"
'';
};
}
# ./oci-images.nix
# simple case: just wraps the operable and adds metadata
let
inherit (inputs) std;
in {
metadata-server = std.lib.ops.mkStandardOCI {
name = "****.amazonaws.com/metadata-server";
operable = cell.entrypoints.metadata-server;
meta = {
description = "The metadata server API.";
};
};
metadata-sync = std.lib.ops.mkStandardOCI {
name = "****.aamazonaws.com/metadata-sync";
operable = cell.entrypoints.metadata-sync;
meta = {
description = "A component that syncs metadata from the Registry (GitHub Repo) to the Sink (Postgres DB).";
};
};
}
# ./deployments.nix
# a prototype implementation using the new haumea matchers
# this generates k8s manifests
# TODO: stabilize and upstream
let
domain = "eks.lw.iog.io";
inherit (inputs) haumea;
inherit (inputs.std) dmerge;
inherit (inputs.nixpkgs) runCommand remarshal;
# Read a YAML file into a Nix datatype using IFD.
# Similar to:
# > builtins.fromJSON (builtins.readFile ./somefile)
# but takes an input file in YAML instead of JSON.
#
# Type:
# Path -> a :: Nix
readYAML = path: let
jsonOutputDrv =
runCommand "from-yaml"
{nativeBuildInputs = [remarshal];}
"remarshal -if yaml -i \"${path}\" -of json -o \"$out\"";
in
fromJSON (readFile jsonOutputDrv);
inherit
(builtins)
fromJSON
;
inherit
(inputs.nixpkgs.lib)
attrNames
elemAt
foldl'
functionArgs
isFunction
isAttrs
length
mapAttrs
mapAttrsRecursiveCond
optionalAttrs
pipe
readFile
setFunctionArgs
mutuallyExclusive
subtractLists
traceSeq
generators
;
mkNames = matches: rec {
env = elemAt matches 0;
name = release + "-backend";
namespace = env + "-" + network;
network = elemAt matches 1;
region = elemAt matches 2;
release = namespace + "-cardanojs";
};
isWrappedComponent = as: as ? __initNomenclature;
loadComponent = f: nomenclature: pipe f [functionArgs (mapAttrs (name: _: nomenclature.${name})) f];
toComponent = f: let
sig1 = attrNames (mkNames null);
sig2 = attrNames (functionArgs f);
excess = subtractLists sig1 sig2;
ok =
isFunction f
&& (! mutuallyExclusive sig2 sig1)
&& (
if excess == []
then true
else
abort ''
Nomenclature currying function signature
${generators.toPretty {multiline = false;} sig2}
has more elements than the available nomenclature
${generators.toPretty {multiline = false;} sig1}.
''
);
wrapped = setFunctionArgs f (functionArgs f) // {__initNomenclature = true;};
in (
if ok
then wrapped
else f
);
instantiateLeaves = nomenclature:
mapAttrsRecursiveCond (c: (!isWrappedComponent c))
(p: f:
if isWrappedComponent f
then loadComponent f nomenclature
else f);
mkComponents = root: nomenclature: let
inherit (dmerge) chainMerge chainable;
components = instantiateLeaves nomenclature root.components;
in {
WithBase = chainable root.base;
WithRegion = chainable components.Region;
WithNamespace = chainable components.Namespace;
WithCardanoStack = chainable components.CardanoStack;
};
loadEnvimontentNix = matches: args: let
nomenclature = mkNames matches;
Components = mkComponents args.root nomenclature;
in
haumea.lib.loaders.default {
inherit domain;
inherit (inputs.nixpkgs) lib;
inherit (args) root;
inherit (dmerge) update append updateOn chainMerge;
inherit Components;
};
loadYaml = _: _: readYAML;
loadMaybeComponent = _: args: path: let
f =
haumea.lib.loaders.default {
inherit domain;
inherit (args) root;
inherit (dmerge) update append updateOn;
}
path;
in
toComponent f;
inherit (haumea.lib.transformers) liftDefault;
in
haumea.lib.load {
src = ./deployments;
transformer = liftDefault;
loader = [
(haumea.matchers.regex ''^.+\.(yaml|yml)'' loadYaml)
(haumea.matchers.regex ''^(.+)-(.+)@(.+)\.nix$'' loadEnvimontentNix)
(haumea.matchers.always loadMaybeComponent)
];
}
#./deployments/[email protected]
{
chainMerge,
Components,
}:
with Components;
chainMerge WithBase WithRegion WithNamespace WithCardanoStack {
meta.description = "Live Environment (user-facing) on the Cardano Proprod Chain (Region A)";
templates = {
backend-deployment = {spec.replicas = 1;};
};
}
Oh and the testbed.nix: (which could benefit from https://github.com/nlewo/nix2container/issues/75)
let
inherit (inputs) std;
in {
metadata = let
user = "metadata";
pass = "bar";
in
std.lib.dev.mkArion {
project.name = "metadata-testbed";
services = {
postgres.service = {
image = "postgres";
restart = "always";
ports = ["5432:5432"];
environment = {
POSTGRES_PASSWORD = pass;
POSTGRES_USER = user;
};
};
metadata-sync = {
service = {
useHostStore = true;
depends_on = ["postgres"];
image = cell.oci-images.metadata-sync.imageRefUnsafe;
environment = {
DB_NAME = user;
DB_USER = user;
DB_PASS = pass;
DB_HOST = "postgres";
DB_PORT = "5432";
GIT_URL = "https://github.com/cardano-foundation/cardano-token-registry.git";
GIT_METADATA_FOLDER = "mappings";
};
};
};
metadata-server = {
service = {
useHostStore = true;
depends_on = ["postgres"];
image = cell.oci-images.metadata-server.imageRefUnsafe;
ports = ["8080:8080"];
environment = {
PORT = "8080";
DB_NAME = user;
DB_USER = user;
DB_PASS = pass;
DB_HOST = "postgres";
DB_PORT = "5432";
};
};
};
};
};
}
Each of those can be interacted with via the available actions in std's TUI with the following bock type declaration:
[
(arion "testbed")
# Packaging Layers
(installables "packages") # {ci.build = true;})
(runnables "entrypoints")
(containers "oci-images" {ci.publish = true;})
# Deployments
(helm "deployments" {ci.diff = true;}) # helm is a repo-local block type that isn't upstreamed (yet)
]
How does this look under flakes output? Do we plan on partially supporting flakes output or go with the pure std in the near future?
@Pegasust for flake outputs compliance, you can use a layer of soil:
growOn {}
# soil for nix cli compat
{
# capture targets
n2cImages = std.harvest inputs.self ["myapp" "oci-images"];
# captures actions ( didn't try that yet - let me know :-) )
app = std.harvest inputs.self.__std.actions ["myapp" "oci-images" ];
}
For average end users not familiar with nix CLI, std (or your branded version) may be a better option.
Actions capturing is very cool, though flakes' output is flat, so we'll need to massage it a bit at the harvest level, so we'll need to massage. We could definitely add a blocktype for this.
We'll also need to turn type derivation -> app
Here's a dump that should be straight-forward regarding the current state of actions capturing
Config:
apps = inputs.nixpkgs.lib.recursiveUpdate (std.harvest self [["repo" "apps"]]) (std.harvest inputs.self.__std.actions [["ops" "operable"]]);
Yields a repl result like this
# vvvvvvvvvvvvvvvvvv (runnables "operable")
nix-repl> apps.aarch64-darwin.racker-backend-ops.build.type
"derivation"
This would work too (by just using another layer of soil):
# 1. layer of soil
{
apps = std.harvest self [["repo" "apps"]];
}
# 2. layer of soil
{
apps = std.harvest inputs.self.__std.actions [["ops" "operable"]];
}
If action harvesting is of more interest, we could definitely add that (and its shape-hammering) to paisano-nix/core.