diff --git a/contrib/ci/Jenkinsfile b/contrib/ci/Jenkinsfile new file mode 100644 index 0000000000000000000000000000000000000000..5ecbe2039f35c188aed65db1184db5913dddcaf2 --- /dev/null +++ b/contrib/ci/Jenkinsfile @@ -0,0 +1,27 @@ +pipeline { + agent { label 'gluon-docker' } + environment { + GLUON_SITEDIR = "contrib/ci/minimal-site" + GLUON_TARGET = "x86-64" + BUILD_LOG = "1" + } + stages { + stage('lint') { + steps { + sh 'luacheck package scripts targets' + } + } + stage('docs') { + steps { + sh 'make -C docs html' + } + } + stage('build') { + steps { + sh 'make update' + sh 'test -d /dl_cache && ln -s /dl_cache openwrt/dl || true' + sh 'make -j$(nproc) V=s' + } + } + } +} diff --git a/contrib/ci/jenkins-community-slave/Dockerfile b/contrib/ci/jenkins-community-slave/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..1ada00f9f39f41c42c42ba79560cb298cbe20d5f --- /dev/null +++ b/contrib/ci/jenkins-community-slave/Dockerfile @@ -0,0 +1,33 @@ +FROM gluon + +USER root + +# this is needed to install default-jre-headless in debian slim images +RUN mkdir -p /usr/share/man/man1 + +RUN apt-get update && apt-get install -y default-jre-headless curl python3 python3-pip python3-sphinx git +RUN pip3 install jenkins-webapi sphinx_rtd_theme + +# Get docker-compose in the agent container +RUN mkdir -p /home/jenkins +RUN mkdir -p /var/lib/jenkins +RUN mkdir -p /remoting +RUN chown gluon /home/jenkins +RUN chown gluon /var/lib/jenkins +RUN chown gluon /remoting + +# Start-up script to attach the slave to the master +ADD slave.py /var/lib/jenkins/slave.py + +USER gluon + +WORKDIR /home/jenkins + +ENV JENKINS_URL "https://build.ffh.zone/" +ENV JENKINS_SLAVE_ADDRESS "" +ENV SLAVE_EXECUTORS "1" +ENV SLAVE_LABELS "docker" +ENV SLAVE_WORING_DIR "" +ENV CLEAN_WORKING_DIR "true" + +CMD [ "python3", "-u", "/var/lib/jenkins/slave.py" ] diff --git a/contrib/ci/jenkins-community-slave/README.md b/contrib/ci/jenkins-community-slave/README.md new file mode 100644 index 0000000000000000000000000000000000000000..aebc78e8a0e7cc534cd88328fe92602a1ea352e7 --- /dev/null +++ b/contrib/ci/jenkins-community-slave/README.md @@ -0,0 +1,32 @@ +# Gluon CI using Jenkins + +## Requirements +- Only a host with docker. + +## Architecture + + + +## Installation +You can support the gluon CI with your infrastructure: +1. You need to query @lemoer (freifunk@irrelefant.net) for credentials. +2. He will give you a `SLAVE_NAME` and a `SLAVE_SECRET` for your host. +3. Then go to your docker host and substitute the values for `SLAVE_NAME` and a `SLAVE_SECRET` in the following statements: +``` shell +git clone https://github.com/freifunk-gluon/gluon/ +cd gluon/contrib/ci/jenkins-community-slave/ +docker build -t gluon-jenkins . +mkdir /var/cache/openwrt_dl_cache/ +docker run --detach --restart always \ + -e "SLAVE_NAME=whoareyou" \ + -e "SLAVE_SECRET=changeme" \ + -v /var/cache/openwrt_dl_cache/:/dl_cache +``` +4. Check whether the instance is running correctly: + - Your node should appear [here](https://build.ffh.zone/label/gluon-docker/). + - When clicking on it, Jenkins should state "Agent is connected." like here: + +5. **Your docker container needs to be rebuilt, when the build dependencies of gluon change. So please be aware of that and update your docker container in that case.** + +## Backoff +- If @lemoer is not reachable, please be patient at first if possible. Otherwise contact info@hannover.freifunk.net or join the channel `#freifunkh` on hackint. diff --git a/contrib/ci/jenkins-community-slave/slave.py b/contrib/ci/jenkins-community-slave/slave.py new file mode 100644 index 0000000000000000000000000000000000000000..30455a87a275728c4a6d2d1f6998d9f7388be954 --- /dev/null +++ b/contrib/ci/jenkins-community-slave/slave.py @@ -0,0 +1,103 @@ +from jenkins import Jenkins, JenkinsError, NodeLaunchMethod +import os +import signal +import sys +import urllib.request +import subprocess +import shutil +import requests +import time + +slave_jar = '/var/lib/jenkins/slave.jar' +slave_name = os.environ['SLAVE_NAME'] if os.environ['SLAVE_NAME'] != '' else 'docker-slave-' + os.environ['HOSTNAME'] +jnlp_url = os.environ['JENKINS_URL'] + '/computer/' + slave_name + '/slave-agent.jnlp' +slave_jar_url = os.environ['JENKINS_URL'] + '/jnlpJars/slave.jar' +print(slave_jar_url) +process = None + +def clean_dir(dir): + for root, dirs, files in os.walk(dir): + for f in files: + os.unlink(os.path.join(root, f)) + for d in dirs: + shutil.rmtree(os.path.join(root, d)) + +def slave_create(node_name, working_dir, executors, labels): + j = Jenkins(os.environ['JENKINS_URL'], os.environ['JENKINS_USER'], os.environ['JENKINS_PASS']) + j.node_create(node_name, working_dir, num_executors = int(executors), labels = labels, launcher = NodeLaunchMethod.JNLP) + +def slave_delete(node_name): + j = Jenkins(os.environ['JENKINS_URL'], os.environ['JENKINS_USER'], os.environ['JENKINS_PASS']) + j.node_delete(node_name) + +def slave_download(target): + if os.path.isfile(slave_jar): + os.remove(slave_jar) + + loader = urllib.request.URLopener() + loader.retrieve(os.environ['JENKINS_URL'] + '/jnlpJars/slave.jar', '/var/lib/jenkins/slave.jar') + +def slave_run(slave_jar, jnlp_url): + params = [ 'java', '-jar', slave_jar, '-jnlpUrl', jnlp_url ] + if os.environ['JENKINS_SLAVE_ADDRESS'] != '': + params.extend([ '-connectTo', os.environ['JENKINS_SLAVE_ADDRESS' ] ]) + + if os.environ['SLAVE_SECRET'] == '': + params.extend([ '-jnlpCredentials', os.environ['JENKINS_USER'] + ':' + os.environ['JENKINS_PASS'] ]) + else: + params.extend([ '-secret', os.environ['SLAVE_SECRET'] ]) + return subprocess.Popen(params, stdout=subprocess.PIPE) + +def signal_handler(sig, frame): + if process != None: + process.send_signal(signal.SIGINT) + +signal.signal(signal.SIGINT, signal_handler) +signal.signal(signal.SIGTERM, signal_handler) + +def h(): + print("ERROR!: please specify environment variables") + print("") + print('docker run -e "SLAVE_NAME=test" -e "SLAVE_SECRET=..." jenkins') + +if os.environ.get('SLAVE_NAME') is None: + h() + sys.exit(1) + +if os.environ.get('SLAVE_SECRET') is None: + h() + sys.exit(1) + +def master_ready(url): + try: + r = requests.head(url, verify=False, timeout=None) + return r.status_code == requests.codes.ok + except: + return False + +while not master_ready(slave_jar_url): + print("Master not ready yet, sleeping for 10sec!") + time.sleep(10) + +slave_download(slave_jar) +print('Downloaded Jenkins slave jar.') + +if os.environ['SLAVE_WORING_DIR']: + os.setcwd(os.environ['SLAVE_WORING_DIR']) + +if os.environ['CLEAN_WORKING_DIR'] == 'true': + clean_dir(os.getcwd()) + print("Cleaned up working directory.") + +if os.environ['SLAVE_NAME'] == '': + slave_create(slave_name, os.getcwd(), os.environ['SLAVE_EXECUTORS'], os.environ['SLAVE_LABELS']) + print('Created temporary Jenkins slave.') + +process = slave_run(slave_jar, jnlp_url) +print('Started Jenkins slave with name "' + slave_name + '" and labels [' + os.environ['SLAVE_LABELS'] + '].') +process.wait() + +print('Jenkins slave stopped.') +if os.environ['SLAVE_NAME'] == '': + slave_delete(slave_name) + print('Removed temporary Jenkins slave.') diff --git a/contrib/ci/minimal-site/i18n b/contrib/ci/minimal-site/i18n new file mode 120000 index 0000000000000000000000000000000000000000..870b9aa3205ce0e96aafd2771d8f9e797c837b54 --- /dev/null +++ b/contrib/ci/minimal-site/i18n @@ -0,0 +1 @@ +../../../docs/site-example/i18n/ \ No newline at end of file diff --git a/contrib/ci/minimal-site/modules b/contrib/ci/minimal-site/modules new file mode 120000 index 0000000000000000000000000000000000000000..be20c51e0b49f81d0baca7a6f1925458e8dd8885 --- /dev/null +++ b/contrib/ci/minimal-site/modules @@ -0,0 +1 @@ +../../../docs/site-example/modules \ No newline at end of file diff --git a/contrib/ci/minimal-site/site.conf b/contrib/ci/minimal-site/site.conf new file mode 100644 index 0000000000000000000000000000000000000000..b7cb1ac472488c38b5e3954d80b65899069dbbe1 --- /dev/null +++ b/contrib/ci/minimal-site/site.conf @@ -0,0 +1,154 @@ +-- This is an example site configuration for Gluon v2018.2+ +-- +-- Take a look at the documentation located at +-- https://gluon.readthedocs.io/ for details. +-- +-- This configuration will not work as is. You're required to make +-- community specific changes to it! +{ + -- Used for generated hostnames, e.g. freifunk-abcdef123456. (optional) + -- hostname_prefix = 'freifunk-', + + -- Name of the community. + site_name = 'Continious Integration', + + -- Shorthand of the community. + site_code = 'ci', + + -- 32 bytes of random data, encoded in hexadecimal + -- This data must be unique among all sites and domains! + -- Can be generated using: echo $(hexdump -v -n 32 -e '1/1 "%02x"' </dev/urandom) + domain_seed = 'e9608c4ff338b920992d629190e9ff11049de1dfc3f299eac07792dfbcda341c', + + -- Prefixes used within the mesh. + -- prefix6 is required, prefix4 can be omitted if next_node.ip4 + -- is not set. + prefix4 = '10.0.0.0/20', + prefix6 = 'fd::/64', + + -- Timezone of your community. + -- See https://openwrt.org/docs/guide-user/base-system/system_configuration#time_zones + timezone = 'CET-1CEST,M3.5.0,M10.5.0/3', + + -- List of NTP servers in your community. + -- Must be reachable using IPv6! + -- ntp_servers = {'1.ntp.services.ffxx'}, + + -- Wireless regulatory domain of your community. + regdom = 'DE', + + -- Wireless configuration for 2.4 GHz interfaces. + wifi24 = { + -- Wireless channel. + channel = 1, + + -- ESSID used for client network. + ap = { + ssid = 'gluon-ci-ssid', + -- disabled = true, -- (optional) + }, + + mesh = { + -- Adjust these values! + id = 'ueH3uXjdp', -- usually you don't want users to connect to this mesh-SSID, so use a cryptic id that no one will accidentally mistake for the client WiFi + mcast_rate = 12000, + -- disabled = true, -- (optional) + }, + }, + + -- Wireless configuration for 5 GHz interfaces. + -- This should be equal to the 2.4 GHz variant, except + -- for channel. + wifi5 = { + channel = 44, + outdoor_chanlist = '100-140', + ap = { + ssid = 'gluon-ci-ssid', + }, + mesh = { + -- Adjust these values! + id = 'ueH3uXjdp', + mcast_rate = 12000, + }, + }, + + + -- The next node feature allows clients to always reach the node it is + -- connected to using a known IP address. + next_node = { + -- anycast IPs of all nodes + -- name = { 'nextnode.location.community.example.org', 'nextnode', 'nn' }, + ip4 = '10.0.0.1', + ip6 = 'fd::1', + }, + + mesh = { + vxlan = true, + batman_adv = { + routing_algo = 'BATMAN_IV' + } + }, + + mesh_vpn = { + -- enabled = true, + mtu = 1312, + + fastd = { + -- Refer to https://fastd.readthedocs.io/en/latest/ to better understand + -- what these options do. + + -- List of crypto-methods to use. + methods = {'salsa2012+umac'}, + -- configurable = true, + -- syslog_level = 'warn', + + groups = { + backbone = { + -- Limit number of connected peers to reduce bandwidth. + limit = 1, + + -- List of peers. + peers = { + }, + + }, + }, + }, + + bandwidth_limit = { + -- The bandwidth limit can be enabled by default here. + enabled = false, + + -- Default upload limit (kbit/s). + egress = 200, + + -- Default download limit (kbit/s). + ingress = 3000, + }, + }, + + autoupdater = { + -- Default branch. Don't forget to set GLUON_BRANCH when building! + branch = 'stable', + + -- List of branches. You may define multiple branches. + branches = { + stable = { + name = 'stable', + + -- List of mirrors to fetch images from. IPv6 required! + mirrors = {'http://1.updates.services.ffhl/stable/sysupgrade'}, + + -- Number of good signatures required. + -- Have multiple maintainers sign your build and only + -- accept it when a sufficient number of them have + -- signed it. + good_signatures = 2, + + -- List of public keys of maintainers. + pubkeys = { + }, + }, + }, + }, +} diff --git a/contrib/ci/minimal-site/site.mk b/contrib/ci/minimal-site/site.mk new file mode 120000 index 0000000000000000000000000000000000000000..873f9cb77dab5ae0e47945de0edc8ce4c3b9df92 --- /dev/null +++ b/contrib/ci/minimal-site/site.mk @@ -0,0 +1 @@ +../../../docs/site-example/site.mk \ No newline at end of file