diff --git a/assets/css/style.css b/assets/css/style.css index 1b9b681..4525ed8 100644 --- a/assets/css/style.css +++ b/assets/css/style.css @@ -8,6 +8,11 @@ src: url(../ttf/ubuntu-regular.ttf) format("truetype"); } +@font-face { + font-family: "Ubuntu Mono"; + src: url(../ttf/ubuntu-mono.ttf) format("truetype"); +} + * { font-family: "Ubuntu Regular"; } @@ -19,7 +24,20 @@ ul { code { background-color: #eee; border: 0.01rem solid #999; - padding: 0.1rem; + padding: 0rem 0.1rem 0rem 0.1rem; +} + + +.TINY { + font-size: 0.75rem; +} + +.WARNING { + background-color: #eee; + border: 1px solid #FF0000; + display: block; + padding: 0.5rem; + font-family: "Ubuntu Mono"; } pre { @@ -27,6 +45,7 @@ pre { border: 1px solid #999; display: block; padding: 0.5rem; + font-family: "Ubuntu Mono"; } #table-of-contents { diff --git a/assets/ttf/ubuntu-mono.ttf b/assets/ttf/ubuntu-mono.ttf new file mode 100644 index 0000000..c8add8e Binary files /dev/null and b/assets/ttf/ubuntu-mono.ttf differ diff --git a/blog/on-databases-docker-and-nix.org b/blog/on-databases-docker-and-nix.org index 879288d..2bb6c78 100644 --- a/blog/on-databases-docker-and-nix.org +++ b/blog/on-databases-docker-and-nix.org @@ -66,9 +66,9 @@ now. Not knowing whether a setting changed? Poof, gone too, Nix is fully declarative, which means it identifies *everything* by a sha256 hash in addition to its developer configured name. Nix also serves as a single point of truth, which means even if the user modifies the config files, they will be overwritten before they are used - again. This makes mix ups are impossible. Messy and flat "configuration" files? Solved too, the Nix expression language can be - as flat or as deep as you need it to be, you can create complex APIs with functions and all that jazz. Basically Nix - is awesome! + again. This makes mix ups are impossible. Messy and flat "configuration" files? Solved too, the Nix expression + language can be as flat or as deep as you need it to be, you can create complex APIs with functions and all that + jazz. Basically Nix is awesome! ** Conclusion The take-away from this rant, is that the best course of action is to figure out either how to completely replace diff --git a/blog/scalable-concourseci-with-nomad-and-nix.org b/blog/scalable-concourseci-with-nomad-and-nix.org index 6d1c894..a91dec8 100644 --- a/blog/scalable-concourseci-with-nomad-and-nix.org +++ b/blog/scalable-concourseci-with-nomad-and-nix.org @@ -1,8 +1,8 @@ #+TITLE: Scalable ConcourseCI with Nomad and Nix In this blog post, I will explain to you how you can deploy ConcourseCI on HashiCorp Nomad with fully automatic and -Op-free scaling. We will utilize 3 HashiCorp tools, namely Nomad, Vault, and Consul, then Nix (not necessary, can be -replaced) and finally ConcourseCI itself. +Op-free scaling. We will utilize 3 HashiCorp tools, namely Nomad, Vault, and Consul, then PosgresSQL, Nix (not +necessary, can be replaced) and finally ConcourseCI itself. * Requirements + a functional Nomad installation with Consul and Vault integration @@ -16,6 +16,7 @@ replaced) and finally ConcourseCI itself. + Linux - 5.11.0 + Nix - 2.4pre20201205_a5d85d0 + ConcourseCI - 7.0.0 + + PostgreSQL - TODO * Overview Our goal is to be able to add a Nomad node to the cluster and have ConcourseCI automatically expand to that node, (We @@ -28,8 +29,8 @@ replaced) and finally ConcourseCI itself. constraints are met. #+END_QUOTE - A ConcourseCI worker node, needs it's own key pair, the best case scenario would be, that we would generate this key - pair, every time a worker node is brought up, and store it in Vault. Fortunately this is possible with a ~pre-start~ + A ConcourseCI worker node needs it's own key pair, the best case scenario would be, that we would generate this key + pair, every time a worker node is brought up and store it in Vault. Fortunately this is possible with a ~pre-start~ task, and Consul Template. \\ That's about it when it comes to the special and interesting bits of this post, so if you already know how to do this, @@ -42,6 +43,444 @@ replaced) and finally ConcourseCI itself. to structure it like so, but you are free to change it around, the only thing that you need to keep the same, is to have a directory with files representing the individual worker nodes, such as =concourse/workers/=. - #+BEGIN_VERBATIM +*** Structure + - [[*concourse][concourse]] + - [[web][web]] + - [[*db][db]] + - [[workers][workers]] + - [[*][]] + +**** concourse + Nothing here, just a folder for the other secrets + +***** web + - tsa_host_key - key used for tsa (communication between a web and a worker node) + - tsa_host_key_pub + - session_signing key + - local_user_name - username of the administrator user + - local_user_pass - password of the administrator user + +***** db + - password + - user + - database + - root_user + - root_password + +***** workers + Holds dynamically generated secrets, of all the workers nodes + +***** + - private_key - the worker's private key + - public_key - the worker's public key, used for authentication when connecting to a web node + +*** Policies + We'll need 3 policies, =concourse-web-policy=, =concourse-worker-policy= and =concourse-db-policy=. + + #+TITLE: concourse-db-policy.hcl + #+BEGIN_SRC hcl + path "kv/data/concourse/db" { + capabilities = ["read"] + } + #+END_SRC + + #+TITLE: concourse-web-policy.hcl + #+BEGIN_SRC hcl + path "kv/data/concourse/workers/*" { + capabilities = ["read"] + } + + path "kv/metadata/concourse/workers" { + capabilities = ["list"] + } + + path "kv/data/concourse/web" { + capabilities = ["read"] + } + + path "kv/data/concourse/db" { + capabilities = ["read"] + } + #+END_SRC + + #+TITLE: concourse-worker-policy.hcl + #+BEGIN_SRC hcl + path "kv/data/concourse/workers/*" { + capabilities = ["read", "update", "delete"] + } + + path "kv/data/concourse/web" { + capabilities = ["read"] + } + #+END_SRC + +*** Application + Create the =web= secrets. - #+END_VERBATIM + #+BEGIN_SRC shell-script + concourse generate-key -t rsa -f ./session_signing_key + _session_signing_key="$(cat session_signing_key)" + concourse generate-key -t ssh -f ./tsa_host_key + _tsa_host_key="$(cat tsa_host_key)" + _tsa_host_key_pub="$(cat tsa_host_key.pub)" + #+END_SRC + + Upload them. + + #+BEGIN_SRC shell-script + vault kv put concourse/web \ + session_signing_key="$_session_signing_key" \ + tsa_host_key="$_tsa_host_key" \ + tsa_host_key_pub="$_tsa_host_key_pub" \ + local_user_pass="changeme" \ + local_user_name="changeme" + #+END_SRC + + Manually specify and upload the secrets for PostgreSQL. + + #+BEGIN_SRC shell-script + vault kv put concourse/db \ + password="changeme" \ + user="changeme" \ + database="changeme" \ + root_user="changeme" \ + root_password="changeme" + #+END_SRC + + #+BEGIN_TINY + The policy file expects the path to be prefixed ~kv/~ if you're using KVv1, with v2 it expects ~kv/data/~. That's + not the case for the ~vault kv~ subcommand, because it automatically prepends the corrects prefix. + #+END_TINY + +** Nomad Web Job + The basic idea of this job is that we start 2 tasks, one for PostgreSQL and the other one for a ConcourseCI, this + could be clustered, but it's out of the scope of this post. + + #+BEGIN_WARNING + The paths in =read=-like go template calls, must be prefixed with =kv/data= while the =list=-like calls, must be + prefixed with =kv/metadata= if you're using KVv2. I figured that out by inspecting the =vault kv= subcommand with + the =-output-curl-string= flag. + #+END_WARNING + + #+BEGIN_SRC hcl + job "concourse-ci-web" { + datacenters = ["homelab-1"] + type = "service" + + group "svc" { + count = 1 + + network { + mode ="bridge" + + port "db" { + to = "5432" + } + port "http" { + static = "8080" + to = "8080" + } + port "tsa" { + static = "2222" + to = "2222" + } + } + + service { + name = "concourse-web" + port = "http" + + check { + type = "http" + path = "/" + interval = "2s" + timeout = "2s" + } + } + + service { + name = "concourse-tsa" + port = "2222" + } + + service { + name = "concourse-db" + port = "db" + } + + task "db" { + driver = "docker" + + config { + image = "magicrb/postgresql@sha256:changeme" + ports = ["db"] + + volumes = [ + "secrets/main.sh:/data/scripts/main.sh", + ] + } + + vault { + policies = ["concourse-db-policy"] + } + + template { + data = <> /data/postgresql/pg_hba.conf + cat << EOD >> /data/postgresql/postgresql.conf + listen_addresses = '0.0.0.0' + password_encryption = md5 + EOD + fi + EOF + destination = "${NOMAD_SECRETS_DIR}/main.sh" + } + } + + task "web" { + driver = "docker" + + config { + image = "concourse/concourse@sha256:changeme" + command = "web" + ports = ["http", "tsa"] + } + + vault { + policies = ["concourse-web-policy"] + } + + restart { + attempts = 5 + delay = "15s" + } + + template { + data = <=. You can actually get the host's hostname from + the =node.unique.name= environment variable, which is not available to the actual code running the in container, but + only in the =template= stanzas. We therefore save it's content into a real environment variable. After that it's + quite simple. \\ + + I'll add the simplified script, which generates and saves the generated secrets. The container must run as a + pre-start task, which is not a sidecar, so that it completes before the main task starts. Template evaluation happens + at runtime, so the secrets will be properly resolved. + + #+BEGIN_SRC shell-script + concourse generate-key -t ssh -f /worker_key + + _worker_key="$(cat /worker_key)" + _worker_key_pub="$(cat /worker_key.pub)" + echo -e "${_worker_key//$'\n'/\\\\n}" > /worker_key + echo -e "${_worker_key_pub//$'\n'/\\\\n}" > /worker_key.pub + + + JSON_FMT='{"public_key":"%s","private_key":"%s"}' + printf "$JSON_FMT" "$(< /worker_key.pub)" "$(< /worker_key)" > secret.json + + vault kv put kv/concourse/workers/blowhole @secret.json + #+END_SRC + + The Bash substitutions are there only to avoid depending on another program like =sed=, which could do it too, and it + would be readable. I also opted for using JSON file, because I was worried I might hit the maximum argument length. \ + + One thing to note is that I haven't yet figured out a way to dynamically get the address of one of the available + Vault instances. So for now, it's okay to hardcode it in. + + #+BEGIN_SRC hcl + job "concourse-ci-worker" { + datacenters = ["homelab-1"] + type = "system" + + group "svc" { + count = 1 + + network { + mode = "bridge" + } + + task "create-secret" { + driver = "docker" + + config { + image = "useyourown" + } + + vault { + policies = ["concourse-worker-policy"] + } + + lifecycle { + sidecar = false + hook = "prestart" + } + + template { + data = < + Require all denied + Options SymLinksIfOwnerMatch + + + + + Require all granted + Options +Indexes +FollowSymlinks + DirectoryIndex index.html + + \ No newline at end of file diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000..63f705a --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1 @@ +apachectl start -f $_apache_cfg -D FOREGROUND diff --git a/flake.lock b/flake.lock index e05db26..9931dfd 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "emacs-htmlize": { + "flake": false, + "locked": { + "lastModified": 1597563983, + "narHash": "sha256-wiRnlWKYQSvQijrymSkEbsW3581LOeuTItkxvTgHXDE=", + "owner": "hniksic", + "repo": "emacs-htmlize", + "rev": "49205105898ba8993b5253beec55d8bddd820a70", + "type": "github" + }, + "original": { + "owner": "hniksic", + "repo": "emacs-htmlize", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1614447571, @@ -29,10 +45,27 @@ "type": "github" } }, + "rlib": { + "locked": { + "lastModified": 1615407344, + "narHash": "sha256-b9fQG73Znv3x4UENEhe75EXDH8x1RcIz6NXj05OHBmY=", + "ref": "master", + "rev": "ea13b638781b3c2a5baf97149873f51a1927d89c", + "revCount": 11, + "type": "git", + "url": "https://gitea.redalder.org/RedAlder/rlib" + }, + "original": { + "type": "git", + "url": "https://gitea.redalder.org/RedAlder/rlib" + } + }, "root": { "inputs": { + "emacs-htmlize": "emacs-htmlize", "nixpkgs": "nixpkgs", - "org-thtml": "org-thtml" + "org-thtml": "org-thtml", + "rlib": "rlib" } } }, diff --git a/flake.nix b/flake.nix index c73e425..8e0696b 100644 --- a/flake.nix +++ b/flake.nix @@ -5,38 +5,135 @@ url = "github:juanjosegarciaripoll/org-thtml"; flake = false; }; + emacs-htmlize = { + url = "github:hniksic/emacs-htmlize"; + flake = false; + }; + + rlib = { + url = "git+https://gitea.redalder.org/RedAlder/rlib"; + flake = true; + }; }; - outputs = { self, nixpkgs, org-thtml }: + outputs = { self, nixpkgs, org-thtml, emacs-htmlize, ... }@inputs: let supportedSystems = [ "x86_64-linux" "i686-linux" "aarch64-linux" ]; forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f system); + + rlib = inputs.rlib.lib { + inherit nixpkgs; + system = "x86_64-linux"; + packages = { + nixpkgs = { + config = {}; + versions = { + stable = inputs.nixpkgs; + }; + }; + custom = {}; + }; + self = rlib; + }; + + websiteBase = pkgs: + pkgs.stdenv.mkDerivation { + name = "magic_rb-website"; + version = "0.1"; + src = ./.; + nativeBuildInputs = [ pkgs.emacs ]; + + buildPhase = '' + cp ${org-thtml}/ox-thtml.el ./ox-thtml.el + cp ${emacs-htmlize}/htmlize.el ./htmlize.el + mkdir tmp && export HOME=$(pwd)/tmp + emacs --script ./make.el + + find public_html -name 'sitemap.*' -exec rm {} \; + ''; + + installPhase = '' + mkdir -p $out + cp -r public_html/* $out + ''; + }; in { website = forAllSystems (system: let pkgs = import nixpkgs { inherit system; }; - in - pkgs.stdenv.mkDerivation { - name = "magic_rb-website"; - version = "0.1"; - src = ./.; - nativeBuildInputs = [ pkgs.emacs ]; - - buildPhase = '' - cp ${org-thtml}/ox-thtml.el ./ox-thtml.el - mkdir tmp && export HOME=$(pwd)/tmp - emacs --script ./make.el - - find public_html -name 'sitemap.*' -exec rm {} \; - ''; - - installPhase = '' - mkdir -p $out - cp -r public_html/* $out - ''; - } + websiteBase pkgs ); + + dockerImages = with rlib.dockerTools; { + apache = buildLayeredImage + ({ nixpkgs, custom, rlib }: + with rlib.dockerTools; + let + shadow = makeShadow { + withNixbld = false; + users = [ + { + name = "www-data"; + uid = "5000"; + gid = "5000"; + home = "/var/empty"; + shell = "${nixpkgs.stable.bash}/bin/bash"; + description = "Apache HTTPD user"; + } + ]; + groups = [ + { + name = "www-data"; + id = 5000; + } + ]; + }; + ca-certificates = makeCerts { + certs = []; + }; + apache = nixpkgs.stable.apache.override { + proxySupport = false; + sslSupport = false; + http2Support = false; + ldapSupport = false; + libxml2Support = false; + brotliSupport = false; + }; + entrypoint = nixpkgs.stable.writeShellScriptBin "entrypoint.sh" (builtins.readFile ./docker/entrypoint.sh); + website = nixpkgs.stable.runCommandNoCCLocal "website" {} '' + mkdir -p $out/var/ + ln -s ${websiteBase nixpkgs.stable} $out/var/www + ''; + logs = nixpkgs.stable.runCommandNoCCLocal "logs" {} '' + mkdir -p $out/var/log/apache2 + ''; + + in + { + name = "magic_rb-website-apache"; + tag = "latest"; + + contents = [ + entrypoint + shadow + ca-certificates + makeTmp + makeBasicBin + website + logs + ]; + + config = with nixpkgs.stable; { + Entrypoint = [ "${dumb-init}/bin/dumb-init" "--" "/bin/entrypoint.sh" ]; + + Env = [ + "PATH=${lib.makeBinPath [ busybox apacheHttpd bash ]}" + "_apache_cfg=${./docker/apache.cfg}" + ]; + }; + }); + }; }; } diff --git a/make.el b/make.el index 66de274..a3010ee 100644 --- a/make.el +++ b/make.el @@ -4,6 +4,7 @@ ;;; Code: (load-file "./ox-thtml.el") +;; (load-file "./htmlize.el") (require 'org) (require 'ox) (require 'ox-html) @@ -22,7 +23,12 @@ ;; DOES NOT WORK https://github.com/alphapapa/unpackaged.el#export-to-html-with-useful-anchors -(setq org-export-with-sub-superscripts '{}) +(setq org-export-with-sub-superscripts '{} + org-export-headline-levels 6 + ; emacs-htmlize does not work when Emacs is ran in =script= mode unfortunately + ;; org-html-html5-fancy t + ;; org-html-htmlize-output-type 'inline-css + ) (defvar org-publish-project-alist) (setq org-publish-project-alist @@ -36,7 +42,8 @@ :with-date nil :html-template ,(templated-html-load-template "templates/blog.html") :publishing-function org-html-publish-to-templated-html - :headline-levels 4 + ;; :htmlized-source t + :headline-levels 5 :exclude "^\\(index.org\\|*.inc\\)" :auto-preamble t :auto-sitemap t