big update based on real wokr
This commit is contained in:
@@ -1,56 +0,0 @@
|
||||
// TODO: PLACEHOLDER — replace with the real estate topology.
|
||||
//
|
||||
// This is where the extracted platform diagrams land. The shape below is
|
||||
// illustrative only: it shows how a mocked dependency, a real service and a
|
||||
// remote system are meant to sit together, not what the system actually is.
|
||||
//
|
||||
// The intended end state is that this file stops being hand-written and is
|
||||
// generated from the running cluster, so the diagram becomes a report of what
|
||||
// exists rather than a drawing of what was once intended.
|
||||
digraph estate {
|
||||
rankdir=LR
|
||||
bgcolor="#0a0e17"
|
||||
fontname="Helvetica"
|
||||
node [fontname="Helvetica" fontsize=11 style=filled color="#1e2a4a" fontcolor="#e8eaf0" shape=box]
|
||||
edge [fontname="Helvetica" fontsize=9 fontcolor="#8892a8" color="#4a5568"]
|
||||
|
||||
label="Estate topology — PLACEHOLDER"
|
||||
labelloc=t
|
||||
fontsize=16
|
||||
fontcolor="#ffc107"
|
||||
|
||||
subgraph cluster_new {
|
||||
label="New"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
api [label="service under work\n(real: built and hot-reloaded)" fillcolor="#1a3a1a" fontcolor="#00c853"]
|
||||
}
|
||||
|
||||
subgraph cluster_core {
|
||||
label="Core (mocked)"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
svc_a [label="upstream service\n(mock: canned responses)" fillcolor="#121829"]
|
||||
db [label="datastore\n(mock)" fillcolor="#121829" shape=cylinder]
|
||||
}
|
||||
|
||||
subgraph cluster_legacy {
|
||||
label="Legacy estate (mocked)"
|
||||
style=dashed
|
||||
color="#1e2a4a"
|
||||
fontcolor="#8892a8"
|
||||
|
||||
batch [label="batch drop\n(mock: writes files on a timer)" fillcolor="#121829"]
|
||||
}
|
||||
|
||||
remote [label="external system\n(remote: ExternalName,\nreachable only from a VDI)" fillcolor="#3a1a1a" fontcolor="#ffc107" shape=octagon]
|
||||
|
||||
api -> svc_a [label="HTTP"]
|
||||
api -> db [label="query"]
|
||||
api -> batch [label="file handoff" style=dashed]
|
||||
api -> remote [label="only when reachable" style=dashed color="#ffc107"]
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 14.1.2 (0)
|
||||
-->
|
||||
<!-- Title: estate Pages: 1 -->
|
||||
<svg width="579pt" height="362pt"
|
||||
viewBox="0.00 0.00 579.00 362.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 357.62)">
|
||||
<title>estate</title>
|
||||
<polygon fill="#0a0e17" stroke="none" points="-4,4 -4,-357.62 575.41,-357.62 575.41,4 -4,4"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="285.7" y="-334.42" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#ffc107">Estate topology — PLACEHOLDER</text>
|
||||
<g id="clust1" class="cluster">
|
||||
<title>cluster_new</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="8,-130.12 8,-210.12 199,-210.12 199,-130.12 8,-130.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="103.5" y="-190.92" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">New</text>
|
||||
</g>
|
||||
<g id="clust2" class="cluster">
|
||||
<title>cluster_core</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="357.33,-172.12 357.33,-318.12 534.83,-318.12 534.83,-172.12 357.33,-172.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-298.92" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Core (mocked)</text>
|
||||
</g>
|
||||
<g id="clust3" class="cluster">
|
||||
<title>cluster_legacy</title>
|
||||
<polygon fill="#0a0e17" stroke="#1e2a4a" stroke-dasharray="5,2" points="341.83,-84.12 341.83,-164.12 551.33,-164.12 551.33,-84.12 341.83,-84.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.58" y="-144.92" font-family="Helvetica,sans-Serif" font-size="16.00" fill="#8892a8">Legacy estate (mocked)</text>
|
||||
</g>
|
||||
<!-- api -->
|
||||
<g id="node1" class="node">
|
||||
<title>api</title>
|
||||
<polygon fill="#1a3a1a" stroke="#1e2a4a" points="191,-174.12 16,-174.12 16,-138.12 191,-138.12 191,-174.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="103.5" y="-159.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">service under work</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="103.5" y="-145.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#00c853">(real: built and hot-reloaded)</text>
|
||||
</g>
|
||||
<!-- svc_a -->
|
||||
<g id="node2" class="node">
|
||||
<title>svc_a</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="526.83,-282.12 365.33,-282.12 365.33,-246.12 526.83,-246.12 526.83,-282.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-267.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">upstream service</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-253.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">(mock: canned responses)</text>
|
||||
</g>
|
||||
<!-- api->svc_a -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>api->svc_a</title>
|
||||
<path fill="none" stroke="#4a5568" d="M149.07,-174.56C167.54,-182.07 189.23,-190.7 209,-198.12 258.25,-216.6 270.12,-222.85 320.75,-237.12 331.43,-240.13 342.71,-243 353.94,-245.67"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="353.12,-249.07 363.66,-247.92 354.71,-242.25 353.12,-249.07"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="255.88" y="-234.24" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">HTTP</text>
|
||||
</g>
|
||||
<!-- db -->
|
||||
<g id="node3" class="node">
|
||||
<title>db</title>
|
||||
<path fill="#121829" stroke="#1e2a4a" d="M480.7,-223.81C480.7,-226.22 465.18,-228.18 446.08,-228.18 426.97,-228.18 411.45,-226.22 411.45,-223.81 411.45,-223.81 411.45,-184.43 411.45,-184.43 411.45,-182.02 426.97,-180.06 446.08,-180.06 465.18,-180.06 480.7,-182.02 480.7,-184.43 480.7,-184.43 480.7,-223.81 480.7,-223.81"/>
|
||||
<path fill="none" stroke="#1e2a4a" d="M480.7,-223.81C480.7,-221.39 465.18,-219.43 446.08,-219.43 426.97,-219.43 411.45,-221.39 411.45,-223.81"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-207.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">datastore</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-193.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">(mock)</text>
|
||||
</g>
|
||||
<!-- api->db -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>api->db</title>
|
||||
<path fill="none" stroke="#4a5568" d="M191.32,-168.36C257.83,-177.73 346.91,-190.28 399.91,-197.75"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="399.4,-201.22 409.79,-199.15 400.38,-194.29 399.4,-201.22"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="255.88" y="-185.69" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">query</text>
|
||||
</g>
|
||||
<!-- batch -->
|
||||
<g id="node4" class="node">
|
||||
<title>batch</title>
|
||||
<polygon fill="#121829" stroke="#1e2a4a" points="537.33,-128.12 354.83,-128.12 354.83,-92.12 537.33,-92.12 537.33,-128.12"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-113.17" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">batch drop</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-99.67" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#e8eaf0">(mock: writes files on a timer)</text>
|
||||
</g>
|
||||
<!-- api->batch -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>api->batch</title>
|
||||
<path fill="none" stroke="#4a5568" stroke-dasharray="5,2" d="M191.32,-144.39C237.67,-138.13 294.98,-130.39 343.4,-123.85"/>
|
||||
<polygon fill="#4a5568" stroke="#4a5568" points="343.61,-127.36 353.05,-122.55 342.67,-120.42 343.61,-127.36"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="255.88" y="-143.94" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">file handoff</text>
|
||||
</g>
|
||||
<!-- remote -->
|
||||
<g id="node5" class="node">
|
||||
<title>remote</title>
|
||||
<polygon fill="#3a1a1a" stroke="#1e2a4a" points="571.41,-21.74 571.41,-52.5 497.99,-74.24 394.17,-74.24 320.75,-52.5 320.75,-21.74 394.17,0 497.99,0 571.41,-21.74"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-46.92" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">external system</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-33.42" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">(remote: ExternalName,</text>
|
||||
<text xml:space="preserve" text-anchor="middle" x="446.08" y="-19.92" font-family="Helvetica,sans-Serif" font-size="11.00" fill="#ffc107">reachable only from a VDI)</text>
|
||||
</g>
|
||||
<!-- api->remote -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>api->remote</title>
|
||||
<path fill="none" stroke="#ffc107" stroke-dasharray="5,2" d="M149.55,-137.68C167.9,-130.34 189.37,-121.97 209,-114.87 254.45,-98.43 305.32,-81.49 348.17,-67.64"/>
|
||||
<polygon fill="#ffc107" stroke="#ffc107" points="349,-71.05 357.44,-64.65 346.85,-64.39 349,-71.05"/>
|
||||
<text xml:space="preserve" text-anchor="middle" x="255.88" y="-117.57" font-family="Helvetica,sans-Serif" font-size="9.00" fill="#8892a8">only when reachable</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 6.9 KiB |
@@ -2,6 +2,7 @@
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<link rel="icon" href="data:,">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>rig — local environment installer</title>
|
||||
<style>
|
||||
@@ -237,13 +238,9 @@
|
||||
<div class="nav-backdrop"></div>
|
||||
<nav>
|
||||
<a href="#start">Start here</a>
|
||||
<a href="#steps">The steps</a>
|
||||
<a href="#install">Installation</a>
|
||||
<a href="#environments">Environments</a>
|
||||
<a href="#overlays">Overlays</a>
|
||||
<a href="#profiles">Profiles</a>
|
||||
<a href="#registry">Registry</a>
|
||||
<a href="#architecture">Architecture</a>
|
||||
<a href="#troubleshooting">Troubleshooting</a>
|
||||
</nav>
|
||||
|
||||
@@ -251,114 +248,68 @@
|
||||
|
||||
<section class="section" id="start">
|
||||
<h2>Start here</h2>
|
||||
<p class="lede">A runnable local model of a large, regulated estate — legacy and new side by side.</p>
|
||||
<div class="prose">
|
||||
<p>rig builds a disposable Kubernetes environment on your machine so you can
|
||||
explore how a system fits together without needing access to any of it. Its
|
||||
job is <b>onboarding and exploration</b>, not a production replica.</p>
|
||||
<p>rig builds a disposable Kubernetes environment on your machine, so you can
|
||||
see how a system fits together without access to any of it. It is for
|
||||
<b>onboarding and exploration</b>, not a production replica.</p>
|
||||
|
||||
<p>Most services in it are deliberately <b>not real</b>. What has to be faithful
|
||||
is the topology — the names, the ports, the dependency order, who can reach whom,
|
||||
and how it fails. The workloads themselves are noise. This is what makes the
|
||||
whole estate fit on a laptop: a real 20-service platform will not fit even once
|
||||
on 14 GB, but mocks are about 30 MB each, so three faithful copies do.</p>
|
||||
<p>Most services in it are <b>not real</b>. What has to be faithful is the
|
||||
topology: names, ports, dependency order, who can reach whom, and how it fails.
|
||||
That is what makes a whole estate fit on a laptop. A real 20-service platform
|
||||
does not fit once in 14 GB; stubs of about 30 MB each fit three times.</p>
|
||||
|
||||
<h3>The only prerequisite</h3>
|
||||
<p><b>Docker.</b> No curl, no jq, no python, no apt repositories to configure.</p>
|
||||
<pre><code><span class="c"># then, in the environment directory:</span>
|
||||
make check <span class="c"># is this machine ready? reports, never fixes</span>
|
||||
make deps <span class="c"># install the pinned toolchain</span>
|
||||
<pre><code><span class="c"># in the environment directory:</span>
|
||||
make check <span class="c"># is this machine ready? reports, never changes anything</span>
|
||||
make deps <span class="c"># install the pinned toolchain; safe to re-run</span>
|
||||
make cluster up <span class="c"># cluster + registry + addons; ports derive by themselves</span>
|
||||
</code></pre>
|
||||
<p>Read <code>make check</code> before <code>make deps</code>. It never changes
|
||||
anything — it prints what it found and, at the end, the steps it cannot perform
|
||||
for you.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="steps">
|
||||
<h2>The steps</h2>
|
||||
<p class="lede">Start to finish, in order, with what each one actually does.</p>
|
||||
<div class="prose">
|
||||
|
||||
<h3>1 · make check</h3>
|
||||
<p>Asks whether this machine is ready. It <b>changes nothing</b> — it
|
||||
reports what it found and, at the end, the things only a human can do
|
||||
(anything needing <code>sudo</code>, or a Windows-side restart). Read it
|
||||
before installing anything; it is faster than discovering the same problems
|
||||
one failure at a time.</p>
|
||||
<pre><code>make check</code></pre>
|
||||
|
||||
<h3>2 · make deps</h3>
|
||||
<p>Installs the pinned toolchain — only what is missing — and tells you
|
||||
if its directory is not on PATH yet. Running it twice is safe.</p>
|
||||
<pre><code>make deps</code></pre>
|
||||
|
||||
<h3>3 · make cluster up</h3>
|
||||
<p>Builds the cluster, starts its registry and installs the profile's
|
||||
addons — there is nothing else to run first. It prints what the profile
|
||||
locks in <i>before</i> spending the time, because the kind config is
|
||||
fixed at creation and cannot be changed afterwards.</p>
|
||||
<p>Re-running is safe and, more importantly, <b>convergent</b>: if a first
|
||||
attempt was interrupted before the CNI was installed, running it again
|
||||
finishes the job rather than reporting "already exists" and leaving every
|
||||
node permanently NotReady.</p>
|
||||
<pre><code>make cluster up <span class="c"># built-in defaults — no profile needed</span>
|
||||
make cluster up PROFILE=mirror <span class="c"># after copying env.d/mirror.env.example: cached registry</span>
|
||||
<p>Read <code>make check</code> before <code>make deps</code>: it lists what is
|
||||
missing, including the steps only you can do, such as anything needing
|
||||
<code>sudo</code> or a Windows-side restart.</p>
|
||||
|
||||
<h3>make cluster up</h3>
|
||||
<p>It prints what the kind config locks in before spending the time, because
|
||||
that config is fixed when the cluster is created. Re-running is safe and
|
||||
continues where an interrupted run stopped, rather than reporting that the
|
||||
cluster already exists and leaving every node <code>NotReady</code>.</p>
|
||||
<pre><code>make cluster up <span class="c"># built-in defaults — no profile needed</span>
|
||||
make cluster up PROFILE=mirror <span class="c"># images through an internal registry</span>
|
||||
make cluster up OVERLAY=examples/data <span class="c"># an overlay: what runs, kept outside rig</span>
|
||||
make cluster reset <span class="c"># destroy and rebuild — how an edited kind config takes effect</span>
|
||||
make cluster reset <span class="c"># destroy and rebuild: how an edited kind config takes effect</span>
|
||||
</code></pre>
|
||||
|
||||
<h3>4 · make docs</h3>
|
||||
<p>Serves this page from a throwaway container. Works with no cluster and
|
||||
no toolchain, which is deliberate: these pages are the instructions for
|
||||
building everything else, so they cannot depend on it.</p>
|
||||
<pre><code>make docs</code></pre>
|
||||
|
||||
<h3>Checking on things</h3>
|
||||
<dl>
|
||||
<dt>make cluster list</dt><dd>Every cluster on the machine, its memory cost and its port block. The usual reason a new one will not start is an old one you forgot about; <code>make cluster free</code> frees them without deleting.</dd>
|
||||
<dt>make check</dt><dd>Short: host, toolchain, and whether this cluster fits, its ports, registry and addons. Details only appear when something needs attention; <code>make check all</code> prints every one.</dd>
|
||||
<dt>make check mem</dt><dd>Memory in depth: what caps it, how far it really climbs, and on WSL the <code>.wslconfig</code> backup and restore.</dd>
|
||||
<dt>make cluster list</dt><dd>Every cluster on the machine, its memory cost and its port block. A new cluster usually fails to start because of an old one; <code>make cluster free</code> stops them without deleting.</dd>
|
||||
<dt>make check</dt><dd>Host, toolchain, and whether this cluster fits, its ports, registry and addons. Only problems print; <code>make check all</code> prints everything.</dd>
|
||||
<dt>make check mem</dt><dd>Memory in depth: what caps it, how far it climbs, and on WSL the <code>.wslconfig</code> backup and restore.</dd>
|
||||
<dt>make docs</dt><dd>Serves these pages from a throwaway container. It needs no cluster and no toolchain, because these are the instructions for building both.</dd>
|
||||
</dl>
|
||||
|
||||
<h3>Running more than one</h3>
|
||||
<p>Name another overlay, or copy the directory and rename it. Cluster name,
|
||||
context, image tags and the port block all follow the folder name — the
|
||||
overlay's, or rig's — so the second environment collides with nothing and
|
||||
neither one's teardown can reach the other.</p>
|
||||
<pre><code>OVERLAY=local/platform-v2 make cluster up
|
||||
cp -r rig ../platform-v3 && cd ../platform-v3 && make cluster up
|
||||
</code></pre>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="install">
|
||||
<h2>Installation</h2>
|
||||
<p class="lede">A container installs onto the host and then gets out of the way.</p>
|
||||
<div class="graph-container">
|
||||
<a href="viewer.html?src=graphs/01-install.svg"><img src="graphs/01-install.svg" alt="Installation flow"></a>
|
||||
<a href="viewer.html?src=graphs/01-install.svg"><img src="graphs/01-install.svg" alt="How the installer works"></a>
|
||||
</div>
|
||||
<div class="prose">
|
||||
<p>The installer is a container, not a shell script, for a specific reason: a
|
||||
stock slim Debian has no <code>curl</code>, no <code>wget</code>, no
|
||||
<code>jq</code>, no <code>python3</code> and <b>no CA bundle</b>. A shell
|
||||
installer could not make a verified HTTPS request, let alone check one. The
|
||||
container carries its own toolchain, so the host needs nothing but Docker.</p>
|
||||
|
||||
<p>The cluster never runs inside that container. Everything it installs —
|
||||
kind, kubectl, tilt, jq — runs natively afterwards, so nothing pays a
|
||||
container tax during daily work.</p>
|
||||
|
||||
<h3>Pinned and verified</h3>
|
||||
<p>Every tool is a single binary fetched at a pinned version and checked
|
||||
against a published SHA256. Node images are pinned <b>by digest</b>, so
|
||||
upgrading kind cannot silently move your Kubernetes version.</p>
|
||||
|
||||
<h3>Not every machine should get cluster tooling</h3>
|
||||
<p>A managed or corporate-issued machine — the kind that holds the access
|
||||
you cannot get anywhere else — is not somewhere to install development
|
||||
tools by default. So the toolchain comes in two tiers:</p>
|
||||
<p>The installer is a container, not a shell script, because a stock slim Debian
|
||||
has no curl, wget, jq, python3 or CA bundle: a shell installer could not make a
|
||||
verified HTTPS request, let alone check one. The container carries its own
|
||||
toolchain, so the host needs only Docker. Nothing runs inside it afterwards —
|
||||
kind, kubectl, tilt and jq run natively.</p>
|
||||
|
||||
<p>Every tool is a single binary at a pinned version, checked against a published
|
||||
SHA256. Node images are pinned <b>by digest</b>, so upgrading kind cannot move
|
||||
your Kubernetes version.</p>
|
||||
|
||||
<h3>Two tiers</h3>
|
||||
<p>A managed or corporate-issued machine is not somewhere to install development
|
||||
tools by default, so installing cluster tooling is a decision rather than a side
|
||||
effect:</p>
|
||||
<table>
|
||||
<tr><th>tier</th><th>installs</th><th>for</th></tr>
|
||||
<tr><td><code>core</code></td><td>kubectl, jq</td><td>talk to a cluster someone else runs</td></tr>
|
||||
@@ -367,11 +318,6 @@ cp -r rig ../platform-v3 && cd ../platform-v3 && make cluster up
|
||||
<pre><code>make deps core <span class="c"># kubectl and jq only — nothing that creates a cluster</span>
|
||||
make deps <span class="c"># dev, the default</span>
|
||||
</code></pre>
|
||||
<p>Testing <i>in situ</i> on a managed machine is still possible — install
|
||||
the <code>dev</code> tier deliberately when you need it. The point is that
|
||||
it should be a decision rather than a side effect of installing.</p>
|
||||
<p>The documentation itself needs neither tier: <code>make docs</code>
|
||||
wants only Docker.</p>
|
||||
|
||||
<h3>Air-gapped</h3>
|
||||
<pre><code>make deps image full <span class="c"># bakes every binary into the image</span>
|
||||
@@ -384,185 +330,129 @@ docker load < rig.tgz && make cluster up PROFILE=offline <span clas
|
||||
|
||||
<section class="section" id="environments">
|
||||
<h2>Environments</h2>
|
||||
<p class="lede">One folder is one environment — an overlay's, or rig's own. Copy it, rename it, run it.</p>
|
||||
<div class="graph-container">
|
||||
<a href="viewer.html?src=graphs/02-environment.svg"><img src="graphs/02-environment.svg" alt="Environment derivation"></a>
|
||||
</div>
|
||||
<div class="prose">
|
||||
<p>Running several versions of a system at once means several clusters on one
|
||||
machine, not several machines. Everything that could collide is derived from
|
||||
the folder name (the overlay's when one is named):</p>
|
||||
<p>One folder is one environment. Running several versions of a system at once
|
||||
means several clusters on one machine, and everything that could collide is
|
||||
derived from the folder name (the overlay's, when one is named):</p>
|
||||
<dl>
|
||||
<dt>cluster + context</dt><dd><code>platform-v2/</code> builds <code>platform-v2</code> on <code>kind-platform-v2</code>.</dd>
|
||||
<dt>port block</dt><dd>Ten ports from a hash of the name, in the 20000+ range — clear of 80, 443, 3000, 5432, 8000 and 8080.</dd>
|
||||
<dt>port block</dt><dd>Ten ports from a hash of the name, above 20000 — clear of 80, 443, 3000, 5432, 8000 and 8080.</dd>
|
||||
<dt>registry + images</dt><dd>Named after the environment, so two copies never share one.</dd>
|
||||
</dl>
|
||||
<p>Two copies therefore never collide, and neither one's
|
||||
<code>make cluster down</code> can touch the other. <code>make check</code>
|
||||
shows the block; <code>bash ctrl/ports.sh persist</code> freezes it into
|
||||
<code>ctrl/.env</code> if you want it fixed rather than derived.</p>
|
||||
<p>So two copies never collide, and neither one's <code>make cluster down</code>
|
||||
can touch the other. <code>make check</code> shows the block;
|
||||
<code>bash ctrl/ports.sh persist</code> freezes it into <code>ctrl/.env</code>.</p>
|
||||
|
||||
<h3>Configuration layers</h3>
|
||||
<p>Weakest first, later wins: built-in defaults → pinned versions → a
|
||||
profile, if you name one → the overlay's <code>rig.env</code> →
|
||||
<code>ctrl/.env</code> → the environment. So
|
||||
<code>make cluster up PROFILE=<name></code> always beats every file.</p>
|
||||
<p>Weakest first, later wins: built-in defaults → pinned versions → a profile, if
|
||||
you name one → the overlay's <code>rig.env</code> → <code>ctrl/.env</code> → the
|
||||
caller's environment. So a value on the command line always beats every file.</p>
|
||||
|
||||
<h3>Profiles</h3>
|
||||
<p>A profile says how this machine reaches the world. rig needs none.</p>
|
||||
<table>
|
||||
<tr><th>example</th><th>registry</th><th>for</th></tr>
|
||||
<tr><td>none</td><td><code>local</code></td><td>the built-in defaults</td></tr>
|
||||
<tr><td><code>mirror.env.example</code></td><td><code>mirror</code></td><td>images through an internal registry</td></tr>
|
||||
<tr><td><code>offline.env.example</code></td><td><code>local</code></td><td>air-gapped</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>Registry modes</h3>
|
||||
<table>
|
||||
<tr><th>mode</th><th>what it does</th></tr>
|
||||
<tr><td><code>none</code></td><td>images are built straight into the node</td></tr>
|
||||
<tr><td><code>local</code></td><td>a registry container wired into the cluster</td></tr>
|
||||
<tr><td><code>mirror</code></td><td>that container as a pull-through cache of the corporate registry</td></tr>
|
||||
<tr><td><code>remote</code></td><td>no local container; pull direct with an <code>imagePullSecret</code></td></tr>
|
||||
</table>
|
||||
<p><code>mirror</code> is what a locked-down network looks like: images come from
|
||||
the corporate registry, you do not hammer it, and you keep working when the
|
||||
connection drops. It is also only reachable from where that registry is routable —
|
||||
from a laptop, use <code>local</code> or <code>offline</code>.</p>
|
||||
<p><b>Corporate CA trust is the usual failure.</b> It has to reach three places:
|
||||
the host Docker daemon, every node's containerd (nodes do not inherit host trust),
|
||||
and any in-cluster client. Set <code>REGISTRY_CA_FILE</code>;
|
||||
<code>make check</code> reports which is still missing. Without it the error is
|
||||
<code>x509: certificate signed by unknown authority</code>.</p>
|
||||
|
||||
<h3>LoadBalancer services</h3>
|
||||
<p>Real manifests use <code>type: LoadBalancer</code>. On a bare local cluster
|
||||
those Services sit at <code>EXTERNAL-IP <pending></code> forever, with no
|
||||
error: the deployment looks healthy and is unreachable. The <code>metallb</code>
|
||||
addon fixes that, so upstream manifests work unchanged. Its address pool is
|
||||
derived from the cluster's Docker network at install time, because Docker picks
|
||||
that subnet and it differs between machines.</p>
|
||||
<p>Those addresses are reachable from the Linux side, including inside WSL. A
|
||||
browser on Windows has no route to them: use the ingress host ports instead.</p>
|
||||
|
||||
<h3>Networking and memory</h3>
|
||||
<p>The cluster uses kind's built-in networking, which does enforce standard
|
||||
NetworkPolicy — verified against a no-policy control. A pluggable CNI was tried
|
||||
and removed: it added policy features nothing here needs, in exchange for a
|
||||
slower boot and one more thing to get right at creation time.</p>
|
||||
<p>Every cluster is a running container tree whether you use it or not.
|
||||
<code>make cluster list</code> shows what each costs, and
|
||||
<code>make cluster free</code> stops the others without deleting them.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="overlays">
|
||||
<h2>Overlays</h2>
|
||||
<p class="lede">What runs lives outside rig — rig reads it and never writes into it.</p>
|
||||
<div class="prose">
|
||||
<p>An overlay is one folder, kept outside rig's version control, holding a
|
||||
use case. Every piece is optional:</p>
|
||||
<p>An overlay is one folder, kept outside rig's version control, holding a use
|
||||
case. rig reads it and never writes into it. Every piece is optional:</p>
|
||||
<table>
|
||||
<tr><th>in the overlay</th><th>what rig does with it</th></tr>
|
||||
<tr><td><code>rig.env</code></td><td>a config layer: addons, namespaces, images — anything a profile could set</td></tr>
|
||||
<tr><td><code>k8s/overlays/dev/</code></td><td>the manifests the dev loop applies</td></tr>
|
||||
<tr><td><code>kind-config.yaml.tpl</code></td><td>the cluster's shape, when it needs its own (mounts, ports)</td></tr>
|
||||
<tr><td><code>addons/<name>.sh</code></td><td>addons, found before rig's own</td></tr>
|
||||
<tr><td><code>addons/<name>.sh</code></td><td>addons, found before rig's own; each applies its own <code><name>.yaml</code></td></tr>
|
||||
<tr><td><code>Tiltfile</code></td><td>the workload's half of the dev loop, included by rig's</td></tr>
|
||||
</table>
|
||||
<pre><code>cp -r examples/starter local/myenv <span class="c"># local/ is gitignored</span>
|
||||
OVERLAY=local/myenv make cluster up
|
||||
</code></pre>
|
||||
<p>With none named, rig runs its own <code>examples/starter</code>.
|
||||
<code>examples/data</code> carries postgres, redis and airflow as an
|
||||
overlay's own addons. A project can also carry rig at
|
||||
<code><project>/rig/</code> and be the overlay itself, with a
|
||||
three-line forwarding Makefile — see <code>docs/notes/overlay.md</code>.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="profiles">
|
||||
<h2>Profiles</h2>
|
||||
<p class="lede">How this machine reaches the world — optional; rig needs none.</p>
|
||||
<div class="prose">
|
||||
<table>
|
||||
<tr><th>example</th><th>registry</th><th>for</th></tr>
|
||||
<tr><td><i>none</i></td><td>local</td><td>the built-in defaults; no profile needed</td></tr>
|
||||
<tr><td><code>mirror.env.example</code></td><td>mirror</td><td>images through an internal registry</td></tr>
|
||||
<tr><td><code>offline.env.example</code></td><td>local</td><td>air-gapped</td></tr>
|
||||
</table>
|
||||
|
||||
<div class="note"><p><b>The kind config cannot be re-applied.</b> Edit
|
||||
<code>ctrl/k8s/kind-config.yaml.tpl</code> (or the overlay's own); it takes effect when the cluster is created. <code>cluster up</code> prints what it
|
||||
locks in before spending the time, and <code>make cluster reset</code> is
|
||||
the way out.</p></div>
|
||||
|
||||
<h3>LoadBalancer services</h3>
|
||||
<p>Real manifests use <code>type: LoadBalancer</code>, because a real
|
||||
cluster has one. On a bare local cluster those Services sit at
|
||||
<code>EXTERNAL-IP <pending></code> forever, with no error anywhere —
|
||||
the deployment looks healthy and simply is not reachable.</p>
|
||||
<p>The <code>metallb</code> addon fixes that, so the same manifests work
|
||||
here as upstream and nothing has to be rewritten to NodePort. Its address
|
||||
pool is derived from the cluster's Docker network at install time rather
|
||||
than hardcoded, because Docker picks that subnet and it differs between
|
||||
machines.</p>
|
||||
<div class="note"><p><b>Where those addresses are reachable from.</b> The
|
||||
pool lives on the Docker bridge, so LoadBalancer IPs work from the Linux
|
||||
side — including from inside WSL. A browser on Windows has no route to
|
||||
them. Use the ingress host ports for anything you need to open in a
|
||||
browser.</p></div>
|
||||
|
||||
<h3>Networking</h3>
|
||||
<p>The cluster uses kind's built-in networking, which <b>does</b> enforce
|
||||
standard NetworkPolicy — verified against a no-policy control, not assumed.
|
||||
The widely repeated claim that it accepts policies and silently ignores
|
||||
them is out of date.</p>
|
||||
<p>A pluggable CNI was tried and removed: it only added
|
||||
GlobalNetworkPolicy, policy tiers and egress-CIDR rules, none of which are
|
||||
needed yet, in exchange for a slower boot and one more thing that has to be
|
||||
right at creation time. Worth revisiting only when a policy the built-in
|
||||
cannot express actually comes up.</p>
|
||||
|
||||
<h3>Memory</h3>
|
||||
<p>Every cluster is a running container tree whether you are using it or not.
|
||||
<code>make cluster list</code> shows what exists and what it costs;
|
||||
<code>make cluster free</code> stops the others without deleting them.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="registry">
|
||||
<h2>Registry</h2>
|
||||
<p class="lede">Local, cached, or straight to the corporate registry.</p>
|
||||
<div class="prose">
|
||||
<table>
|
||||
<tr><th>mode</th><th>what it does</th></tr>
|
||||
<tr><td><code>none</code></td><td>images are built straight into the node</td></tr>
|
||||
<tr><td><code>local</code></td><td>a registry container wired into the cluster</td></tr>
|
||||
<tr><td><code>mirror</code></td><td>that container as a <b>pull-through cache</b> of the corporate registry</td></tr>
|
||||
<tr><td><code>remote</code></td><td>no local container; pull direct with an imagePullSecret</td></tr>
|
||||
</table>
|
||||
<p><code>mirror</code> is what a locked-down network actually looks like:
|
||||
images originate from the corporate registry, you do not hammer it, and you
|
||||
keep working when the connection drops.</p>
|
||||
|
||||
<div class="note"><p><b>The corporate CA will bite you.</b> A corporate
|
||||
registry is usually behind an internal CA, and trust has to reach
|
||||
<b>three</b> places: the host Docker daemon, every cluster node's containerd
|
||||
(nodes do <i>not</i> inherit host trust), and any in-cluster client. Set
|
||||
<code>REGISTRY_CA_FILE</code> and <code>make check</code> reports which is
|
||||
still missing. The symptom otherwise is an opaque
|
||||
<code>x509: certificate signed by unknown authority</code>.</p></div>
|
||||
|
||||
<p>Reachability also depends on where you are: if the registry is only
|
||||
routable from a managed workspace, <code>mirror</code> and <code>remote</code>
|
||||
will not resolve from a laptop at all. That is what <code>local</code> and
|
||||
<code>offline</code> are for.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="architecture">
|
||||
<h2>Architecture</h2>
|
||||
<p class="lede">The estate being modelled.</p>
|
||||
<div class="note"><p><b>TODO — placeholder.</b> The diagram below is
|
||||
illustrative only: it shows how a mocked dependency, a service under active
|
||||
work, and an unreachable remote system sit together. It is not the real
|
||||
topology. Replace <code>docs/graphs/03-architecture.dot</code> with the
|
||||
extracted platform diagrams, then run <code>make docs graphs</code>.</p></div>
|
||||
<div class="graph-container">
|
||||
<a href="viewer.html?src=graphs/03-architecture.svg"><img src="graphs/03-architecture.svg" alt="Estate topology (placeholder)"></a>
|
||||
</div>
|
||||
<div class="prose">
|
||||
<p>Each component is one of three things, and switching between them should be
|
||||
a one-line change rather than a rewrite:</p>
|
||||
<code>examples/data</code> carries postgres, redis and airflow as an overlay's own
|
||||
addons. A project can also carry rig at <code><project>/rig/</code> and be
|
||||
the overlay itself, with a three-line forwarding Makefile — see
|
||||
<code>docs/notes/overlay.md</code>.</p>
|
||||
|
||||
<h3>What runs in it</h3>
|
||||
<p>Each component is one of three things, and switching between them is a
|
||||
one-line change:</p>
|
||||
<dl>
|
||||
<dt>real</dt><dd>Built from source and hot-reloaded. The thing you are actually working on — usually exactly one.</dd>
|
||||
<dt>real</dt><dd>Built from source and hot-reloaded. What you are working on — usually exactly one.</dd>
|
||||
<dt>mock</dt><dd>A generic stub with canned responses. Everything you do not care about today.</dd>
|
||||
<dt>remote</dt><dd>No pod at all: a Service of type ExternalName pointing at the real system. In-cluster DNS resolves identically, so callers never change.</dd>
|
||||
<dt>remote</dt><dd>No pod at all: a Service of type <code>ExternalName</code> pointing at the real system. In-cluster DNS resolves identically, so callers never change.</dd>
|
||||
</dl>
|
||||
<p>The intended end state is that this diagram is <b>generated from the
|
||||
running cluster</b> rather than drawn by hand — so it becomes a report of
|
||||
what exists instead of a picture of what was once intended.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="section" id="troubleshooting">
|
||||
<h2>Troubleshooting</h2>
|
||||
<p class="lede">The failures that are hard to diagnose from their symptoms.</p>
|
||||
<div class="prose">
|
||||
<h3>Tilt stops noticing file changes</h3>
|
||||
<p>Almost always <code>inotify</code> limits, and it fails <i>silently</i> —
|
||||
nothing errors, changes just stop being picked up. Defaults on WSL are far too
|
||||
low. <code>make check</code> reports it and prints the fix.</p>
|
||||
<p>Almost always inotify limits, and it fails silently: nothing errors, changes
|
||||
just stop being picked up. Defaults on WSL are far too low.
|
||||
<code>make check</code> reports it and prints the fix.</p>
|
||||
|
||||
<h3>Cluster creation dies halfway with a port error</h3>
|
||||
<p>Docker reports <code>failed to bind host port … address already in use</code>
|
||||
partway through creating the cluster. Run <code>make check</code> first — it
|
||||
checks every port in this environment's block before anything is built.</p>
|
||||
partway through creating the cluster. <code>make check</code> tests every port in
|
||||
this environment's block before anything is built.</p>
|
||||
|
||||
<h3>Every node stays NotReady</h3>
|
||||
<p>Usually a cluster created with the default CNI disabled but the real CNI
|
||||
never installed — typically an interrupted first run. Just run
|
||||
<code>make cluster up</code> again: it converges rather than exiting early, and
|
||||
will finish the missing steps.</p>
|
||||
<p>Usually a cluster created with the default CNI disabled and the real CNI never
|
||||
installed, after an interrupted first run. Run <code>make cluster up</code> again:
|
||||
it finishes the missing steps instead of stopping because the cluster exists.</p>
|
||||
|
||||
<h3>x509: certificate signed by unknown authority</h3>
|
||||
<p>Corporate CA trust has not reached one of the three places it needs to be.
|
||||
See <a href="#registry">Registry</a>.</p>
|
||||
<p>Corporate CA trust has not reached one of the three places it needs to be. See
|
||||
<a href="#environments">Environments</a>.</p>
|
||||
|
||||
<h3>kubectl says the context does not exist</h3>
|
||||
<p>The cluster can exist while its context does not — a reset or a switched
|
||||
|
||||
@@ -4,6 +4,12 @@
|
||||
|
||||
Each addon is its own idempotent script — adding one is adding a file, not editing a dispatcher. `ADDONS` names them, in install order; the overlay's `addons/<name>.sh` is found before rig's `ctrl/addons/<name>.sh`, and every one runs from rig's `ctrl/` with `RIG_CTRL` exported, wherever its file lives (see [overlay.md](overlay.md)).
|
||||
|
||||
## The manifest beside the script
|
||||
|
||||
An addon's Kubernetes objects are a plain `<name>.yaml` beside its `<name>.sh`, not a heredoc inside it: the YAML reads, diffs and lints as YAML, and the script keeps only what YAML cannot do (secrets made once, a database created, waiting for a rollout). The script renders it with `render_manifest "$ADDON_DIR/<name>.yaml"` (`lib/config.sh`) and pipes it to `kubectl apply`.
|
||||
|
||||
`render_manifest` fills `${NAME}` and `${NAME:-default}` from the loaded config or the script's own variables, and fails on a name nothing sets, so a typo is an error rather than an empty image. `$VAR` and `$(VAR)` are left alone: they are the container's, as in postgres' `pg_isready -U $POSTGRES_USER`. selftest checks that every name a manifest uses is set somewhere. `ADDON_DIR` is taken before the script's `cd` into rig's `ctrl/`: `ADDON_DIR="$(cd "$(dirname "$0")" && pwd)"`.
|
||||
|
||||
## What rig ships, and what it does not
|
||||
|
||||
rig's own addons make the *cluster* work, and are useless outside one: metallb, cert-manager, metrics-server. Things a workload happens to need — a database, a cache, a scheduler — are the workload's, and which workload needs which is not rig's business, so they live with the overlay. `examples/data/addons/` has postgres, redis and airflow as a worked example; an overlay that wants them copies them in.
|
||||
|
||||
@@ -34,14 +34,37 @@ A port held by THIS environment's own cluster is not a clash; it is the thing wo
|
||||
|
||||
The ports are extracted with a second grep rather than `tr -d ':->'`: in tr, ':->' is the character RANGE ':' to '>', which does not contain '-', so the trailing dash survives and nothing ever matches.
|
||||
|
||||
## Compact by default
|
||||
|
||||
`make check` prints one line per question — host, toolchain, and for this rig: cluster, memory,
|
||||
ports, registry, addons — and adds detail only where something needs attention (`!` lines, the
|
||||
"held elsewhere" list when memory is tight, the clashing port). `make check all` prints every fact,
|
||||
as the full report did before 2026-09-17. `deps.sh detect all` is the same switch for the host part,
|
||||
so the standalone `rigdeps.sh detect` is short too. Changed because the long report buried the few
|
||||
lines that mattered.
|
||||
## Budget first, configuration after (2026-09-23)
|
||||
|
||||
`make check` answers the question people run it for first, in the shape the adapter rig had
|
||||
(the user preferred it): a memory budget with one row per part, the node overhead, the total
|
||||
against what the machine has and has available; then a one-line verdict (`fits`, `! tight`,
|
||||
`! does not fit`, with the two opposite fixes told apart); then where things stand and the next
|
||||
command — not created, stopped (with the `docker start` line), up (with the Tilt and gateway
|
||||
URLs, and whether this environment's Tilt is already running).
|
||||
|
||||
Below that, the configuration as it is, raw: the host and toolchain (`deps.sh detect`), then
|
||||
one `key value` line each for cluster, context, overlay, profile, nodes, kind config, manifests,
|
||||
ingress, ports (named), registry, addons. `!` lines always print; `make check all` adds every
|
||||
fact (per-port detail, addon list, swap and overcommit notes). `deps.sh detect all` is the same
|
||||
switch for the host part, so the standalone `rigdeps.sh detect` is short too.
|
||||
|
||||
## MEMORY_BUDGET
|
||||
|
||||
rig knows what a node costs (NODE_MB), not what a workload does, so the parts come from the
|
||||
overlay's rig.env: `MEMORY_BUDGET="name=MB ..."`, in the order listed; a 0 is left out, and an
|
||||
entry that is not `name=MB` is a `!` line. rig.env is sourced, so an overlay can size a part by
|
||||
its own knob (poc_rig's airflow follows its mode). When one part outweighs the rest together
|
||||
(and is at least 1 GB), check names it as the first thing to switch off; `MEMORY_BUDGET_TIP`
|
||||
says how, in the overlay's words. No budget: one `?` row says where to add it.
|
||||
|
||||
## Tilt: ours or another
|
||||
|
||||
A Tilt still running after `make cluster down` holds the Tilt port, and the next `make tilt`
|
||||
fails with "address already in use" (2026-09-23, the Workspace). check reads the listener with
|
||||
`ss -p` (our own processes only): a `tilt` whose working directory is this ctrl/ is this
|
||||
environment's Tilt, reported as running with its URL; another `tilt` is a `!` line with its pid,
|
||||
its folder and the `kill` that frees the port. Anything else is the ordinary port clash.
|
||||
|
||||
## overlay and kind config
|
||||
|
||||
|
||||
@@ -8,6 +8,12 @@ out `python -m http.server` too — a minimal Debian has no python3. What it doe
|
||||
have, by definition, is Docker: the single prerequisite rig already demands. So a
|
||||
throwaway nginx container serves a read-only bind mount.
|
||||
|
||||
It runs with a config of its own: one worker (nginx's `auto` starts one per core,
|
||||
twelve on an ordinary laptop, for a handful of static pages), no access log and a
|
||||
quiet entrypoint, so the terminal shows the URL and nothing else unless something
|
||||
fails. The pages declare an empty icon, so the browser does not ask for a
|
||||
`favicon.ico` that is not there.
|
||||
|
||||
## Committed SVGs
|
||||
|
||||
Rendered SVGs are committed alongside their `.dot` sources for the same reason:
|
||||
|
||||
@@ -60,6 +60,9 @@ air-gapped install); an overlay says what runs. `ctrl/.env` is still this
|
||||
machine's, and the caller still wins over everything.
|
||||
|
||||
`rig.env` may not set `PROFILE` or `OVERLAY`: both are chosen before it loads.
|
||||
|
||||
`MEMORY_BUDGET` (and optionally `MEMORY_BUDGET_TIP`) is how an overlay tells `make check` what its
|
||||
workload costs: rig only knows the node. See check.md.
|
||||
The paths it sets (`MANIFESTS_DIR`, `KIND_CONFIG`) are relative to the overlay.
|
||||
`MANIFESTS_DIR=none` means rig applies no manifests and the overlay's Tiltfile
|
||||
does, e.g. when kustomize needs flags.
|
||||
@@ -100,7 +103,8 @@ load_config
|
||||
```
|
||||
|
||||
`OVERLAY_DIR` is set, so an addon can find files beside it
|
||||
(`$(_from_ctrl "$OVERLAY_DIR")/...`). rig's own `ctrl/addons/` holds only what
|
||||
(`$(_from_ctrl "$OVERLAY_DIR")/...`). Its manifest is a `<name>.yaml` beside the script,
|
||||
rendered with `render_manifest` (see [addons.md](addons.md)). rig's own `ctrl/addons/` holds only what
|
||||
makes a cluster work (metallb, cert-manager, metrics-server);
|
||||
`examples/data/addons/` shows workload ones.
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<link rel="icon" href="data:,">
|
||||
<title>Graph Viewer</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; }
|
||||
|
||||
Reference in New Issue
Block a user