diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b3d5992 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.direnv +result* diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0f5e8c3 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +OUT_DIR ?= ../out/ +AUX_DIR ?= ../aux/ +TOP_LIB ?= ../latex/mk/ + +PROJECTS := tex +DEPS_EXTS := tex png pdf + +EXTRA_LATEXMK_ARGS=-pdfxelatex="xelatex --shell-escape %O %S" + +.PHONY: all +all: $(PROJECTS) +clean: + rm -r out aux + +include $(TOP_LIB)/build_project.mk + +$(eval $(call build_projects,$(PROJECTS))) diff --git a/figures/blender.png b/figures/blender.png new file mode 100644 index 0000000..bea1dfc Binary files /dev/null and b/figures/blender.png differ diff --git a/figures/buildbot-view.png b/figures/buildbot-view.png new file mode 100644 index 0000000..25bc7ae Binary files /dev/null and b/figures/buildbot-view.png differ diff --git a/figures/forgejo.png b/figures/forgejo.png new file mode 100644 index 0000000..e1a1912 Binary files /dev/null and b/figures/forgejo.png differ diff --git a/figures/gentoo.png b/figures/gentoo.png new file mode 100644 index 0000000..8034dd9 Binary files /dev/null and b/figures/gentoo.png differ diff --git a/figures/gentoo.svg b/figures/gentoo.svg new file mode 100644 index 0000000..31953c0 --- /dev/null +++ b/figures/gentoo.svg @@ -0,0 +1,24 @@ + + +Gentoo Linux logo (matte) + + + + + + + + + + + + + + + + + + + + + diff --git a/figures/gh-status-ex01.png b/figures/gh-status-ex01.png new file mode 100644 index 0000000..e996cc9 Binary files /dev/null and b/figures/gh-status-ex01.png differ diff --git a/figures/github.png b/figures/github.png new file mode 100644 index 0000000..1d05574 Binary files /dev/null and b/figures/github.png differ diff --git a/figures/github.svg b/figures/github.svg new file mode 100644 index 0000000..dfe3f14 --- /dev/null +++ b/figures/github.svg @@ -0,0 +1,3 @@ + \ No newline at end of file diff --git a/figures/how-standards-prolifirate.png b/figures/how-standards-prolifirate.png new file mode 100644 index 0000000..6923b8e Binary files /dev/null and b/figures/how-standards-prolifirate.png differ diff --git a/figures/how-standards-prolifirate.xcf b/figures/how-standards-prolifirate.xcf new file mode 100644 index 0000000..7f26d23 Binary files /dev/null and b/figures/how-standards-prolifirate.xcf differ diff --git a/figures/llvm.png b/figures/llvm.png new file mode 100644 index 0000000..ddbb4b9 Binary files /dev/null and b/figures/llvm.png differ diff --git a/figures/nix-community.png b/figures/nix-community.png new file mode 100644 index 0000000..dea16fa Binary files /dev/null and b/figures/nix-community.png differ diff --git a/figures/nix-community.svg b/figures/nix-community.svg new file mode 100644 index 0000000..684bc6d --- /dev/null +++ b/figures/nix-community.svg @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + diff --git a/figures/openzfs-logo.png b/figures/openzfs-logo.png new file mode 100644 index 0000000..d38695b Binary files /dev/null and b/figures/openzfs-logo.png differ diff --git a/figures/python-buildbot.png b/figures/python-buildbot.png new file mode 100644 index 0000000..d9afb33 Binary files /dev/null and b/figures/python-buildbot.png differ diff --git a/figures/python-logo.svg b/figures/python-logo.svg new file mode 100644 index 0000000..269bbea --- /dev/null +++ b/figures/python-logo.svg @@ -0,0 +1,265 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/figures/python.png b/figures/python.png new file mode 100644 index 0000000..66ae7f8 Binary files /dev/null and b/figures/python.png differ diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..7f5b0b7 --- /dev/null +++ b/flake.lock @@ -0,0 +1,93 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1727826117, + "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1728909085, + "narHash": "sha256-WLxED18lodtQiayIPDE5zwAfkPJSjHJ35UhZ8h3cJUg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c0b1da36f7c34a7146501f684e9ebdf15d2bebf8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1727825735, + "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1726871744, + "narHash": "sha256-V5LpfdHyQkUF7RfOaDPrZDP+oqz88lTJrMT1+stXNwo=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a1d92660c6b3b7c26fb883500a80ea9d33321be2", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs", + "treefmt-nix": "treefmt-nix" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1729077719, + "narHash": "sha256-zayHqZO9gA1U85c4CPvVSnLV8/cBgc2yVrSKWaKeBUs=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "5307ba60125bb024d7e52d71d582eafd511f3fee", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..3c78b65 --- /dev/null +++ b/flake.nix @@ -0,0 +1,34 @@ +{ + inputs.nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-24.05"; + inputs.flake-parts.url = "github:hercules-ci/flake-parts"; + inputs.treefmt-nix.url = "github:numtide/treefmt-nix"; + + outputs = + { + self, + nixpkgs, + flake-parts, + ... + }@inputs: + flake-parts.lib.mkFlake { inherit inputs; } { + imports = [ inputs.treefmt-nix.flakeModule ]; + systems = [ "x86_64-linux" ]; + perSystem = + { config, pkgs, ... }: + { + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + nixfmt-rfc-style + texliveFull + python3.pkgs.pygments + ]; + }; + + treefmt = { + projectRootFile = "flake.nix"; + programs.nixfmt-rfc-style.enable = true; + programs.latexindent.enable = true; + }; + }; + }; +} diff --git a/tex/main.tex b/tex/main.tex new file mode 100644 index 0000000..3dd6290 --- /dev/null +++ b/tex/main.tex @@ -0,0 +1,357 @@ +\documentclass{beamer} + +\usepackage{graphicx} +\usepackage[outputdir=../aux/tex]{minted} +\usepackage[showboxes,absolute,overlay]{textpos} +\usepackage{tikz} +\usepackage{svg} +\usetikzlibrary{calc,tikzmark} + +\title{buildbot-nix} +\author{Richard Brežák} +\date{2024} + +\usetheme[progressbar=frametitle]{moloch} + +\newcommand\beamercoverdeemph{% + \setbeamercovered{again covered={\opaqueness<1->{30}}}% +} + +\begin{document} +\frame{\titlepage} + +\begin{frame} + \centering + There is no good open-source Nix CI/CD +\end{frame} + +\begin{frame}{Hydra} + \beamercoverdeemph{} + \centering + \begin{itemize} + \item<2>{heavy-weight, keeps track of every store path} + \item<3>{lots of legacy Perl} + \item<4>{hard to deploy, administer and hack on} + \item<5>{\texttt{nixpkgs} scale CI for \texttt{nixpkgs} scale problems} + \item<6,7>{doesn't integrate well with GitHub/Gitea\only<7>{, especially PRs}} + \end{itemize} +\end{frame} + +\begin{frame} + \begin{columns}[c] + \begin{column}{0.49\textwidth} + \centering + I attempted to run Hydra + \end{column} + \begin{column}{0.02\textwidth} + \rule{.1mm}{0.7\textheight} + \end{column} + \begin{column}{0.49\textwidth} + \centering + I've succeeded and I'm still running it + \end{column} + \end{columns} +\end{frame} + +\begin{frame}{GitHub/Gitea Actions} + \beamercoverdeemph{} + \centering + \begin{itemize} + \item<2>{not Nix native} + \item<3>{cannot use the build sandbox generally} + \item<4>{caching is hard to get right or wasteful} + \item<5>{build logs become a jumbled mess \textit{OR} may repeat builds} + \item<6>{if done naively, you will get an OOM} + \end{itemize} +\end{frame} + +\begin{frame} + \includegraphics[width=1.0\textwidth]{../figures/how-standards-prolifirate.png} +\end{frame} + +\begin{frame}[fragile] + \begin{center} + \verb|buildbot-nix| + \end{center} +\end{frame} + + +\begin{frame} + \begin{columns}[c] + \begin{column}{0.49\textwidth} + \centering + I know of buildbot-nix + \end{column} + \begin{column}{0.02\textwidth} + \rule{.1mm}{0.7\textheight} + \end{column} + \begin{column}{0.49\textwidth} + \centering + I've never heard of it + \end{column} + \end{columns} +\end{frame} + +\begin{frame}{buildbot?} + \begin{columns}[c] + \begin{column}{0.5\textwidth} + \beamercoverdeemph{} + \begin{itemize} + \item<1> CI/CD framework + \item<2> written in Python + \item<3> used by many projects + \end{itemize} + \vspace{1.0em} + \uncover<4->{essentially...} \\ + \vspace{1.0em} + \uncover<5->{Jenkins but in Python} + \end{column} + \begin{column}{0.5\textwidth} + \pgfdeclareimage[width=5em]{openzfs}{../figures/openzfs-logo.png} + \pgfdeclareimage[width=5em]{python}{../figures/python.png} + \pgfdeclareimage[width=5em]{gentoo}{../figures/gentoo.png} + \pgfdeclareimage[width=5em]{blender}{../figures/blender.png} + \pgfdeclareimage[width=5em]{llvm}{../figures/llvm.png} + \pgfdeclareimage[width=20em]{nix-community}{../figures/nix-community.png} + \begin{column}<3->{0.5\textwidth} + %% \pgfmathsetseed{2} + \begin{tikzpicture} + %% \makeatletter + %% \pgfmathdeclarefunction{rndx}{0}{ + %% \pgfmathrandom{-1em, 1em} \pgfmathresult + %% } + %% \pgfmathdeclarefunction{rndy}{0}{ + %% \pgfmathrandom{-1em, 1em} \pgfmathresult + %% } + %% \makeatother + %% \draw (0, 0) node {\pgftext{\includesvg[width=1em]{../figures/python-logo.svg}}}; + \draw (-14em, 7em) node {\pgfuseimage{openzfs}}; + \draw (-14em, -7em) node {\pgfuseimage{blender}}; + \draw (-14em, 0) node {\pgfuseimage{gentoo}}; + \draw (-7em, 3.5em) node {\pgfuseimage{python}}; + \draw (-7em, -3.5em) node {\pgfuseimage{llvm}}; + %% \draw (0, 0) node {\pgftext{\includegraphics{silver}}}}; + \end{tikzpicture} + \end{column} + \end{column} + \end{columns} +\end{frame} +\begin{frame}{buildbot!} + \includegraphics[width=1.0\linewidth]{../figures/python-buildbot.png} +\end{frame} + +\begin{frame}{buildbot-nix?} + \beamercoverdeemph{} + \begin{itemize} + \item<1> CI/CD for Nix + \item<2> plugin for \texttt{buildbot} + \item<3> written in Python + \item<4> can run on PRs safely -- configuration is server-side + \end{itemize} + \begin{uncoverenv}<5> + \begin{center} + \Large Aiming for smaller projects + \end{center} + \end{uncoverenv} + \begin{onlyenv}<6> + \begin{tikzpicture}[overlay] + \fill[white, opacity=0.7] (-0.2\paperwidth,1.0\paperheight) rectangle (1.0\paperwidth, -0.2\paperheight); + \draw (0.45\textwidth,0.175\textheight) node {\pgfuseimage{nix-community}}; + \end{tikzpicture} + \end{onlyenv} +\end{frame} + +\begin{frame}[fragile] + \begin{minted}[linenos=true,bgcolor=bg,breaklines,tabsize=2]{nix} +{ + inputs = { ... }; + outputs = { ... }: + { + checks.${system} = { + treefmt = treefmt...check; + packageTest = runCommand "package-test" { } "exit 1"; + }; + formatter.${system} = treefmt...wrapper; + }; +} + \end{minted} + {\tiny (not true Nix)} +\end{frame} + +\begin{frame} + \includegraphics[width=1.0\textwidth]{../figures/gh-status-ex01.png} + + \begin{textblock}{2.5}(12.5,14.5) + {\footnotesize(find the bug)} + \end{textblock} +\end{frame} + +\begin{frame} + \includegraphics[width=1.0\textwidth]{../figures/buildbot-view.png} +\end{frame} + +\begin{frame} + \centering + How do I get this? +\end{frame} + +\begin{frame} + \centering + Easily! +\end{frame} + +\begin{frame}[fragile] + \newcommand<>{\circletikz}[2]{% + \begin{onlyenv}#3 + \begin{tikzpicture}[overlay,remember picture] + \draw[draw = red, line width=0.3mm] let + \p1 = ($(pic cs:#1) - (pic cs:#2)$), + \p2 = ($1/2*(\x1,\y1)$) + in ($(pic cs:#1) - 1/2*(\x1, \y1) + (0, 0.25em)$) ellipse (\x2 and 1em); + \end{tikzpicture}% + \end{onlyenv}% + } + \newcommand<>{\underlinetikz}[2]{% + \begin{onlyenv}#3 + \begin{tikzpicture}[overlay,remember picture] + \draw[draw = red, line width=0.3mm] + ($(pic cs:#1) - (0, 0.125em)$) -- ($(pic cs:#2) - (0, 0.125em)$); + \end{tikzpicture}% + \end{onlyenv}% + } + \begin{minted}[escapeinside=||,beameroverlays,fontsize=\small,linenos=true,bgcolor=bg,breaklines,tabsize=2]{nix} +services.buildbot-nix.master = { + enable = true; + jobReportLimit = |\tikzmark{jobsS}|null|\tikzmark{jobsE}|; + domain = |\tikzmark{domainS}|"buildbot.example.org"|\tikzmark{domainE}|; + workersFile = |\tikzmark{workersS}|"/secret/workers.json"|\tikzmark{workersE}|; + admins = |\tikzmark{adminsS}|["your-nickname"]|\tikzmark{adminsE}|; + authBackend = |\tikzmark{githubS}|"github"|\tikzmark{githubE}|; + |\tikzmark{githubS}|github|\tikzmark{githubE}| = { + |\tikzmark{githubEnableS}|enable = true|\tikzmark{githubEnableE}|; + webhookSecretFile = |\tikzmark{webhookS}|"/secret/github_webhook_secret"|\tikzmark{webhookE}|; + oauthId = |\tikzmark{oauthIdS}|"..."|\tikzmark{oauthIdE}|; + oauthSecretFile = |\tikzmark{oauthSecretS}|"/secret/github_oauth_secret"|\tikzmark{oauthSecretE}|; + topic = |\tikzmark{topicS}|"build-with-buildbot"|\tikzmark{topicE}|; + authType.app = { + secretKeyFile = |\tikzmark{secretKeyS}|"/secret/github_app_private_key.pem"|\tikzmark{secretKeyE}|; + id = |\tikzmark{idS}|881425|\tikzmark{idE}|; + }; + }; +}; + \end{minted} + \underlinetikz<2>{domainS}{domainE} + + \underlinetikz<3>{githubS}{githubE} + \underlinetikz<3>{githubEnableS}{githubEnableE} + + \underlinetikz<4>{workersS}{workersE} + \underlinetikz<4>{webhookS}{webhookE} + \underlinetikz<4>{oauthIdS}{oauthIdE} + \underlinetikz<4>{oauthSecretS}{oauthSecretE} + \underlinetikz<4>{secretKeyS}{secretKeyE} + \underlinetikz<4>{idS}{idE} + + \underlinetikz<5>{topicS}{topicE} + \underlinetikz<5>{jobsS}{jobsE} +\end{frame} + +\begin{frame}[fragile] + \begin{center} + and you have buildbot running + \end{center} + \begin{textblock}{10.0}(5.5,14.5) + {\footnotesize(after configuring \verb|services.nginx.virtualHosts|)} + \end{textblock} +\end{frame} + +\begin{frame} + \centering + What are the features? +\end{frame} + +\newcommand{\imagewithtitle}[2]{% + \begin{minipage}[t][1em]{1.0\linewidth} + \centering #1 + \end{minipage}\\ + \vspace{1.5em} + \includegraphics[width=1.0\linewidth, height=0.8\textheight, keepaspectratio]{#2}% +} + +%% \begin{frame}[fragile] +%% %% \begin{columns} +%% %% \begin{column}{0.5\textwidth} +%% \centering +%% \imagewithtitle{\Large builds \texttt{.\#checks}}{../figures/buildbot-view.png} +%% %% \end{column} +%% %% \begin{column}{0.5\textwidth} +%% \centering +%% \imagewithtitle{\Large status reports}{../figures/gh-status-ex01.png} +%% %% \end{column} +%% %% \end{columns} +%% \end{frame} + +\begin{frame} + \begin{columns} + \begin{column}{0.5\textwidth} + \centering + \imagewithtitle{\Large Gitea/Forgejo}{../figures/forgejo.png} + \end{column} + \begin{column}{0.5\textwidth} + \centering + \imagewithtitle{\Large GitHub}{../figures/github.png} + \end{column} + \end{columns} +\end{frame} + +\begin{frame} + \centering + \imagewithtitle{\Large Cachix}{example-image-a} +\end{frame} + +\begin{frame} + \centering + \imagewithtitle{\Large post build steps}{example-image-a} +\end{frame} + +\begin{frame} + \centering + \imagewithtitle{\Large per-repository settings}{example-image-a} +\end{frame} + +\begin{frame} + \centering + \imagewithtitle{\Large per-repository effects with secrets (soon)}{example-image-a} +\end{frame} + +\begin{frame} + \centering + Questions? +\end{frame} + +%% \begin{frame} +%% \centering +%% what can it do? +%% \end{frame} + +%% \begin{frame}[fragile] +%% \centering +%% \verb|buildbot-nix| will evaluate \verb|#checks| for all systems +%% \end{frame} +%% \begin{frame}[fragile] +%% \centering +%% it will report evaluation failures +%% \end{frame} +%% \begin{frame}[fragile] +%% \centering +%% it will build derivations for supported systems +%% \end{frame} + +\end{document} + +% there is no good opensource CI +% then we built it +% why is hydra not great +% what is it we built, what does it offer +% now there is good opensource CI, check out +% call for action