Blob Blame History Raw
#!/bin/bash -e
# Copyright 2019 Red Hat
# This file is part of copperblue.
#
# Copperblue is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Copperblue is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with copperblue.  If not, see <https://www.gnu.org/licenses/>.

REF=48/658248/2
TAG=$(echo $REF | tr '/' '-')
POD="zuul-runner-services"
SRC=${HOME}/.cache/zuul/zuul
REQ="buildah podman redhat-rpm-config gcc-c++ re2-devel python3-devel git"
REQ="${REQ} bubblewrap"

function log {
    echo -e "$(date "+%H:%m:%S"): \e[33m[+] $1\e[00m"
}

function checkout_ref {
    git fetch https://review.opendev.org/zuul/zuul refs/changes/$REF && \
        git checkout -b $TAG FETCH_HEAD
}

function install_zuul {
    log "Installing zuul runner patch"
    [ -d $SRC ] || mkdir -p $SRC
    git clone https://opendev.org/zuul/zuul $SRC
    pushd $SRC
    checkout_ref
    python3 setup.py develop --user
    popd
}

function update_zuul {
    log "Updating zuul runner code"
    pushd $SRC
    checkout_ref
    popd
}

function prepare_image {
    log "Patching zuul/zuul"
    C=$(buildah from zuul/zuul)
    buildah copy $C $SRC/zuul /usr/local/lib/python3.7/site-packages/zuul/
    buildah commit $C zuul/zuul:$TAG
    buildah delete $C
}

function prepare_conf {
    [ -d ~/.cache/zuul/conf ] || mkdir -p ~/.cache/zuul/conf
    [ -d ~/.cache/zuul/conf/state ] || mkdir -p ~/.cache/zuul/conf/state
    [ -d ~/.cache/zuul/git ] || mkdir -p ~/.cache/zuul/git
    [ -d ~/.cache/zuul/config ] || mkdir -p ~/.cache/zuul/config
    cat <<EOF> ~/.cache/zuul/conf/zuul.conf
[gearman]
server=scheduler

[gearman_server]
start=true

[zookeeper]
hosts=zk

[scheduler]
tenant_config=/etc/zuul/main.yaml
state_dir=/etc/zuul/state

[connection "local"]
driver=git
baseurl=file:///local-git

[web]
listen_address=0.0.0.0
EOF
    if [ -z "${GERRIT_ACCOUNT}" ]; then
        cat <<EOF>> ~/.cache/zuul/conf/zuul.conf
[connection "opendev.org"]
driver=git
baseurl=https://opendev.org
EOF
    else
        cat <<EOF>> ~/.cache/zuul/conf/zuul.conf
[connection "opendev.org"]
driver=gerrit
port=29418
server=review.opendev.org
baseurl=https://review.opendev.org
user=$GERRIT_ACCOUNT
sshkey=/var/lib/zuul/.ssh/id_rsa
canonical_hostname=opendev.org
EOF
        log "Testing gerrit connection"
        ssh -p 29418 -l $GERRIT_ACCOUNT review.opendev.org gerrit version
    fi

    cat <<EOF> ~/.cache/zuul/conf/main.yaml
- tenant:
    name: runner
    source:
      local:
        config-projects:
          - config
      opendev.org:
        config-projects:
          - zuul/zuul-base-jobs
        untrusted-projects:
          - zuul/zuul-jobs:
              include:
                - job
          - openstack/openstack-zuul-jobs
          - openstack/tripleo-ci
          - openstack/tripleo-quickstart
          - openstack/tripleo-quickstart-extras
          - openstack/tripleo-upgrade
          - x/browbeat
          - openstack/tripleo-ha-utils
          - openstack/tripleo-heat-templates
          - openstack/openstack-virtual-baremetal
          - openstack/openstack-ansible-os_tempest
          - openstack/ansible-role-python_venv_build
          - openstack/ansible-config_template
          - openstack/tripleo-common
          - openstack/python-tripleoclient
          - openstack/requirements
          - openstack/ansible-role-container-registry
          - openstack/kolla
          - openstack/tripleo-repos
          - openstack/ansible-role-collect-logs
EOF
    cat <<EOF > ~/.cache/zuul/config/zuul.yaml
- pipeline:
    name: check
    description: local pipelines
    manager: independent
    require: {}
    trigger: {}
    success: {}
    failure: {}
- pipeline:
    name: periodic
    description: local pipelines
    manager: independent
    require: {}
    trigger: {}
    success: {}
    failure: {}
EOF
    pushd ~/.cache/zuul/config/ > /dev/null
    [ -d .git ] || {
        log "Creating config project"
        git init
        git add zuul.yaml
        git commit -m "Initialize config project"
    }
    if ! test -z "$(git status -s)"; then
        log "Updating config project"
        git add *
        git commit -m "Update config project"
    fi
    popd > /dev/null
}

function start_services {
    log "Starting the pod"
    podman pod create --name $POD -p 9000:9000
    podman pod start $POD
    log "Starting the containers"
    podman run --rm --pod $POD -d --name zk --hostname zk zookeeper
    [ -z "${GERRIT_ACCOUNT}" ] || \
        SSH_SHARE="-v ${HOME}/.ssh/id_rsa:/var/lib/zuul/.ssh/id_rsa:z"
    podman run --rm --pod $POD -d --name scheduler --hostname scheduler \
        -v ~/.cache/zuul/conf:/etc/zuul/:z  \
        -v ~/.cache/zuul/:/local-git:z $SSH_SHARE \
        --add-host scheduler:127.0.0.1 \
        --add-host zk:127.0.0.1 \
        localhost/zuul/zuul:$TAG zuul-scheduler -d
    podman run --rm --pod $POD -d --name web --hostname web \
        -v ~/.cache/zuul/conf:/etc/zuul/:z  \
        --add-host scheduler:127.0.0.1 \
        --add-host zk:127.0.0.1 \
        localhost/zuul/zuul:$TAG zuul-web -d
    podman run --rm --pod $POD -d --name merger --hostname merger \
        -v ~/.cache/zuul/conf:/etc/zuul/:z $SSH_SHARE \
        -v ~/.cache/zuul/git:/var/lib/zuul/merger-git:z  \
        -v ~/.cache/zuul/:/local-git:z  \
        --add-host scheduler:127.0.0.1 \
        --add-host zk:127.0.0.1 \
        zuul/zuul zuul-merger -d
}

function wait_for_api {
    log "Waiting for the api"
    for i in $(seq 300); do
        curl -I localhost:9000/api/tenant/runner/project/openstack/tripleo-ci \
             2> /dev/null | grep "HTTP/1.1 200 OK" && ok=1 && break
        echo -n "."
        sleep 1;
    done
    echo
    if [ "$ok" != "1" ]; then
        log "Failed to start the scheduler, logs are:"
        podman logs scheduler
        exit 1
    fi
}

function show_info {
    log "Available jobs:"
    curl http://localhost:9000/api/tenant/runner/pipeline/check/\
project/openstack/tripleo-ci/branch/master/freeze-jobs \
     2> /dev/null | tr ',' '\n' | awk '/name/ { print $2 }' \
     | sed -e 's/^\"/ * /' -e 's/"$//'
    echo
    echo "You can now reproduce a job by running:"
    echo ANSIBLE_STDOUT_CALLBACK=yaml zuul-runner --verbose --tenant runner \
         --api http://localhost:9000/ --project openstack/tripleo-ci \
         --job tripleo-ci-centos-7-standalone \
         execute \
         --nodes ssh:centos-7:NODE_IP:centos:/home/centos -d /tmp/job-workspace
    echo "Stop the servicing by running: podman pod rm -f $POD"
}

rpm -q $REQ &> /dev/null || {
    log "Installing $REQ:"
    sudo dnf install -y $REQ
}

if ! type -p zuul-runner &> /dev/null; then
    install_zuul
fi

if [ "$(cd $SRC; git rev-parse --abbrev-ref HEAD)" != "$TAG" ]; then
    update_zuul
fi

if ! buildah images -nq localhost/zuul/zuul:$TAG &> /dev/null; then
   prepare_image
fi

prepare_conf
if podman pod exists $POD; then
    log "Stopping pod"
    podman pod rm -f $POD
fi
start_services

wait_for_api
show_info

# Note: to prepare the node you need:
# sudo ln -s /home/centos /home/zuul
# sudo mkdir -m 0777 /etc/nodepool
# mkdir -p /home/centos/workspace/logs
# TODO: something to create /etc/ci/mirror_info.sh