PR#46: OpenShift deployment - waiverdb

		`@@ -9,7 +9,11 @@`
		`ARG waiverdb_rpm`
		`COPY $waiverdb_rpm /tmp`
		`# XXX take out epel-testing eventually`
		`- RUN yum -y install --setopt=tsflags=nodocs --enablerepo=epel-testing python-gunicorn /tmp/$(basename $waiverdb_rpm) && yum -y clean all`
		`+ RUN yum -y install --setopt=tsflags=nodocs --enablerepo=epel-testing \`
		`+ python-gunicorn \`
		`+ python-psycopg2 \`
		`+ /tmp/$(basename $waiverdb_rpm) \`
		`+ && yum -y clean all`
		`USER 1001`
		`EXPOSE 8080`
		`ENTRYPOINT gunicorn --bind 0.0.0.0:8080 --access-logfile=- waiverdb.wsgi:app`

Jenkinsfile

file modified

+47

		`@@ -116,6 +116,9 @@`
		`def image = docker.build "factory2/waiverdb:${appversion}", "--build-arg waiverdb_rpm=$el7_rpm ."`
		`image.push()`
		`}`
		`+ /* Save container version for later steps (this is ugly but I can't find anything better...) */`
		`+ writeFile file: 'appversion', text: appversion`
		`+ archiveArtifacts artifacts: 'appversion'`
		`}`
		`} catch (e) {`
		`currentBuild.result = "FAILED"`
		`@@ -126,3 +129,47 @@`
		`throw e`
		`}`
		`}`
		`+ node('fedora') {`
		`+ sh 'sudo dnf -y install /usr/bin/py.test'`
		`+ checkout scm`
		`+ stage('Perform functional tests') {`
		`+ unarchive mapping: ['appversion': 'appversion']`
		`+ def appversion = readFile('appversion').trim()`
		`+ openshift.withCluster('open.paas.redhat.com') {`
		`+ openshift.doAs('openpaas-waiverdb-test-jenkins-credentials') {`
		`+ openshift.withProject('waiverdb-test') {`
		`+ def template = readYaml file: 'openshift/waiverdb-test-template.yaml'`
		`+ def models = openshift.process(template,`
		`+ '-p', "TEST_ID=${env.BUILD_TAG}",`
		`+ '-p', "WAIVERDB_APP_VERSION=${appversion}")`
		`+ def environment_label = "test-${env.BUILD_TAG}"`
		`+ try {`
		`+ def objects = openshift.create(models)`
		`+ echo "Waiting for pods with label environment=${environment_label} to become Ready"`
		`+ def pods = openshift.selector('pods', ['environment': environment_label])`
		`+ timeout(15) {`
		`+ pods.untilEach(3) {`
		`+ def conds = it.object().status.conditions`
		`+ for (int i = 0; i < conds.size(); i++) {`
		`+ if (conds[i].type == 'Ready' && conds[i].status == 'True') {`
		`+ return true`
		`+ }`
		`+ }`
		`+ return false`
		`+ }`
		`+ }`
		`+ def route_hostname = objects.narrow('route').object().spec.host`
		`+ echo "Running tests against https://${route_hostname}/"`
		`+ withEnv(["WAIVERDB_TEST_URL=https://${route_hostname}/"]) {`
		`+ sh 'py.test functional-tests/'`
		`+ }`
		`+ } finally {`
		`+ /* Tear down everything we just created */`
		`+ openshift.selector('dc,deploy,pod,configmap,secret,svc,route',`
		`+ ['environment': environment_label]).delete()`
		`+ }`
		`+ }`
		`+ }`
		`+ }`
		`+ }`
		`+ }`

conf/settings.py.example

file modified

+2 -2

		`@@ -1,8 +1,8 @@`
		# Copy this file to `conf/settings.py` to put it into effect. It overrides the values defined
		# in `waiverdb/config.py`.
		`SECRET_KEY = 'replace-me-with-something-random'`
		`- #SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://dbuser:dbpassword@dbhost:dbport/dbname'`
		`- SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/waiverdb_db.sqlite'`
		`+ #DATABASE_URI = 'postgresql+psycopg2://dbuser:dbpassword@dbhost:dbport/dbname'`
		`+ DATABASE_URI = 'sqlite:////var/tmp/waiverdb_db.sqlite'`
		`JOURNAL_LOGGING = False`
		`#SHOW_DB_URI = False`
		`HOST= '0.0.0.0'`

functional-tests/conftest.py

file added

+32

		`@@ -0,0 +1,32 @@`
		`+`
		`+ # This program is free software; you can redistribute it and/or modify`
		`+ # it under the terms of the GNU General Public License as published by`
		`+ # the Free Software Foundation; either version 2 of the License, or`
		`+ # (at your option) any later version.`
		`+ #`
		`+ # This program is distributed in the hope that it will be useful,`
		`+ # but WITHOUT ANY WARRANTY; without even the implied warranty of`
		`+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the`
		`+ # GNU General Public License for more details.`
		`+ #`
		`+`
		`+ import os`
		`+ import pytest`
		`+ import requests`
		`+`
		`+`
		`+ @pytest.fixture(scope='session')`
		`+ def waiverdb_url():`
		`+ if 'WAIVERDB_TEST_URL' not in os.environ:`
		`+ raise AssertionError('WAIVERDB_TEST_URL=http://example.com/ '`
		`+ 'must be set in the environment')`
		`+ url = os.environ['WAIVERDB_TEST_URL']`
		`+ assert url.endswith('/')`
		`+ return url`
		`+`
		`+`
		`+ @pytest.fixture(scope='session')`
		`+ def requests_session(request):`
		`+ s = requests.Session()`
		`+ request.addfinalizer(s.close)`
		`+ return s`

functional-tests/test_api_v1.py

file added

+17

		`@@ -0,0 +1,17 @@`
		`+`
		`+ # This program is free software; you can redistribute it and/or modify`
		`+ # it under the terms of the GNU General Public License as published by`
		`+ # the Free Software Foundation; either version 2 of the License, or`
		`+ # (at your option) any later version.`
		`+ #`
		`+ # This program is distributed in the hope that it will be useful,`
		`+ # but WITHOUT ANY WARRANTY; without even the implied warranty of`
		`+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the`
		`+ # GNU General Public License for more details.`
		`+ #`
		`+`
		`+`
		`+ def test_get_waivers(waiverdb_url, requests_session):`
		`+ response = requests_session.get(waiverdb_url + 'api/v1.0/waivers/')`
		`+ response.raise_for_status()`
		`+ assert response.json()['data'] == []`

openshift/waiverdb-test-template.yaml

file added

+227

		`@@ -0,0 +1,227 @@`
		`+`
		`+ # Template to produce a new test environment in OpenShift. Uses OpenID Connect`
		`+ # against iddev.fedorainfracloud.org for authentication, and ephemeral storage`
		`+ # for Postgres data.`
		`+ #`
		`+ # To create an environment from the template, process and apply it:`
		`+ # oc process -f openshift/waiverdb-test-template.yaml -p TEST_ID=123 -p WAIVERDB_APP_VERSION=0.1.2.dev24-git.94c0119 \| oc apply -f -`
		`+ # To clean up the environment, use a selector on the environment label:`
		`+ # oc delete dc,deploy,pod,configmap,secret,svc,route -l environment=test-123`
		`+`
		`+ ---`
		`+ apiVersion: v1`
		`+ kind: Template`
		`+ metadata:`
		`+ name: waiverdb-test-template`
		`+ parameters:`
		`+ - name: TEST_ID`
		`+ displayName: Test id`
		`+ description: Short unique identifier for this test run (e.g. Jenkins job number)`
		`+ required: true`
		`+ - name: WAIVERDB_APP_VERSION`
		`+ displayName: WaiverDB application version`
		`+ description: Python version of the WaiverDB application being tested`
		`+ required: true`
		`+ - name: FLASK_SECRET_KEY`
		`+ displayName: Flask secret key`
		`+ generate: expression`
		`+ from: "[\\w]{32}"`
		`+ - name: DATABASE_PASSWORD`
		`+ displayName: Database password`
		`+ generate: expression`
		`+ from: "[\\w]{32}"`
		`+ objects:`
		`+ - apiVersion: v1`
		`+ kind: Secret`
		`+ metadata:`
		`+ name: "waiverdb-test-${TEST_ID}-secret"`
		`+ labels:`
		`+ environment: "test-${TEST_ID}"`
		`+ stringData:`
		`+ flask-secret-key: "${FLASK_SECRET_KEY}"`
		`+ database-password: "${DATABASE_PASSWORD}"`
		`+ # This is the same non-secret config we have committed`
		`+ # as conf/client_secrets.json for using in dev environments.`
		`+ client_secrets.json: \|-`
		`+ {"web": {`
		`+ "redirect_uris": ["http://localhost:8080/"],`
		`+ "token_uri": "https://iddev.fedorainfracloud.org/openidc/Token",`
		`+ "auth_uri": "https://iddev.fedorainfracloud.org/openidc/Authorization",`
		`+ "client_id": "D-e69a1ac7-30fa-4d18-9001-7468c4f34c3c",`
		`+ "client_secret": "qgz8Bzjg6nO7JWCXoB0o8L49KfI5atLF",`
		`+ "userinfo_uri": "https://iddev.fedorainfracloud.org/openidc/UserInfo",`
		`+ "token_introspection_uri": "https://iddev.fedorainfracloud.org/openidc/TokenInfo"}}`
		`+ - apiVersion: v1`
		`+ kind: ConfigMap`
		`+ metadata:`
		`+ name: "waiverdb-test-${TEST_ID}-configmap"`
		`+ labels:`
		`+ environment: "test-${TEST_ID}"`
		`+ data:`
		`+ settings.py: \|-`
		`+ DATABASE_URI = 'postgresql+psycopg2://waiverdb@waiverdb-test-${TEST_ID}-database:5432/waiverdb'`
		`+ PORT = 8080`
		`+ AUTH_METHOD = 'OIDC'`
		`+ OIDC_CLIENT_SECRETS = '/etc/secret/client_secrets.json'`
		`+ - apiVersion: v1`
		`+ kind: Service`
		`+ metadata:`
		`+ name: "waiverdb-test-${TEST_ID}-database"`
		`+ labels:`
		`+ environment: "test-${TEST_ID}"`
		`+ spec:`
		`+ selector:`
		`+ environment: "test-${TEST_ID}"`
		`+ service: database`
		`+ ports:`
		`+ - name: postgresql`
		`+ port: 5432`
		`+ targetPort: 5432`
		`+ - apiVersion: v1`
		`+ kind: DeploymentConfig`
		`+ metadata:`
		`+ name: "waiverdb-test-${TEST_ID}-database"`
		`+ labels:`
		`+ environment: "test-${TEST_ID}"`
		`+ service: database`
		`+ spec:`
		`+ replicas: 1`
		`+ strategy:`
		`+ type: Recreate`
		`+ selector:`
		`+ environment: "test-${TEST_ID}"`
		`+ service: database`
		`+ template:`
		`+ metadata:`
		`+ labels:`
		`+ environment: "test-${TEST_ID}"`
		`+ service: database`
		`+ spec:`
		`+ containers:`
		`+ - name: postgresql`
		`+ image: registry.access.redhat.com/rhscl/postgresql-95-rhel7:latest`
		`+ imagePullPolicy: Always`
		`+ ports:`
		`+ - containerPort: 5432`
		`+ readinessProbe:`
		`+ timeoutSeconds: 1`
		`+ initialDelaySeconds: 5`
		`+ exec:`
		`+ command: [ /bin/sh, -i, -c, "psql -h 127.0.0.1 -U $POSTGRESQL_USER -q -d $POSTGRESQL_DATABASE -c 'SELECT 1'" ]`
		`+ livenessProbe:`
		`+ timeoutSeconds: 1`
		`+ initialDelaySeconds: 30`
		`+ tcpSocket:`
		`+ port: 5432`
		`+ env:`
		`+ - name: POSTGRESQL_USER`
		`+ value: waiverdb`
		`+ - name: POSTGRESQL_PASSWORD`
		`+ valueFrom:`
		`+ secretKeyRef:`
		`+ name: "waiverdb-test-${TEST_ID}-secret"`
		`+ key: database-password`
		`+ - name: POSTGRESQL_DATABASE`
		`+ value: waiverdb`
		`+ triggers:`
		`+ - type: ConfigChange`
		`+ - apiVersion: v1`
		`+ kind: Service`
		`+ metadata:`
		`+ name: "waiverdb-test-${TEST_ID}-web"`
		`+ labels:`
		`+ environment: "test-${TEST_ID}"`
		`+ annotations:`
		`+ service.alpha.openshift.io/dependencies: \|-`
		`+ [{"name": "waiverdb-test-${TEST_ID}-database", "kind": "Service"}]`
		`+ spec:`
		`+ selector:`
		`+ environment: "test-${TEST_ID}"`
		`+ service: web`
		`+ ports:`
		`+ - name: web`
		`+ port: 8080`
		`+ targetPort: 8080`
		`+ - apiVersion: v1`
		`+ kind: Route`
		`+ metadata:`
		`+ name: "waiverdb-test-${TEST_ID}-web"`
		`+ labels:`
		`+ environment: "test-${TEST_ID}"`
		`+ spec:`
		`+ port:`
		`+ targetPort: web`
		`+ to:`
		`+ kind: Service`
		`+ name: "waiverdb-test-${TEST_ID}-web"`
		`+ tls:`
		`+ termination: edge`
		`+ insecureEdgeTerminationPolicy: Redirect`
		`+ - apiVersion: v1`
		`+ kind: DeploymentConfig`
		`+ metadata:`
		`+ name: "waiverdb-test-${TEST_ID}-web"`
		`+ labels:`
		`+ environment: "test-${TEST_ID}"`
		`+ service: web`
		`+ spec:`
		`+ replicas: 2`
		`+ selector:`
		`+ environment: "test-${TEST_ID}"`
		`+ service: web`
		`+ template:`
		`+ metadata:`
		`+ labels:`
		`+ environment: "test-${TEST_ID}"`
		`+ service: web`
		`+ spec:`
		`+ containers:`
		`+ - name: web`
		`+ image: "docker-registry.engineering.redhat.com/factory2/waiverdb:${WAIVERDB_APP_VERSION}"`
		`+ ports:`
		`+ - containerPort: 8080`
		`+ volumeMounts:`
		`+ - name: config-volume`
		`+ mountPath: /etc/waiverdb`
		`+ readOnly: true`
		`+ - name: secret-volume`
		`+ mountPath: /etc/secret`
		`+ readOnly: true`
		`+ env:`
		`+ - name: DATABASE_PASSWORD`
		`+ valueFrom:`
		`+ secretKeyRef:`
		`+ name: "waiverdb-test-${TEST_ID}-secret"`
		`+ key: database-password`
		`+ - name: SECRET_KEY`
		`+ valueFrom:`
		`+ secretKeyRef:`
		`+ name: "waiverdb-test-${TEST_ID}-secret"`
		`+ key: flask-secret-key`
		`+ readinessProbe:`
		`+ timeoutSeconds: 1`
		`+ initialDelaySeconds: 5`
		`+ httpGet:`
		`+ path: /healthcheck`
		`+ port: 8080`
		`+ livenessProbe:`
		`+ timeoutSeconds: 1`
		`+ initialDelaySeconds: 30`
		`+ httpGet:`
		`+ path: /healthcheck`
		`+ port: 8080`
		`+ # Limit to 384MB memory. This is probably not enough but it is`
		`+ # necessary in the current environment to allow for 2 replicas and`
		`+ # rolling updates, without hitting the (very aggressive) memory quota.`
		`+ resources:`
		`+ limits:`
		`+ memory: 384Mi`
		`+ volumes:`
		`+ - name: config-volume`
		`+ configMap:`
		`+ name: "waiverdb-test-${TEST_ID}-configmap"`
		`+ - name: secret-volume`
		`+ secret:`
		`+ secretName: "waiverdb-test-${TEST_ID}-secret"`
		`+ triggers:`
		`+ - type: ConfigChange`

waiverdb/app.py

file modified

+37 -2

		`@@ -10,6 +10,7 @@`
		`# GNU General Public License for more details.`

		`import os`
		`+ import urlparse`

		`from flask import Flask`
		`from sqlalchemy import event`
		`@@ -35,6 +36,27 @@`
		`app.config.from_object(default_config_obj)`
		`config_file = os.environ.get('WAIVERDB_CONFIG', default_config_file)`
		`app.config.from_pyfile(config_file)`
		`+ if os.environ.get('SECRET_KEY'):`
		`+ app.config['SECRET_KEY'] = os.environ['SECRET_KEY']`
		`+`
		`+`
		`+ def populate_db_config(app):`
		`+ # Take the application-level DATABASE_URI setting, plus (optionally)`
		`+ # a DATABASE_PASSWORD from the environment, and munge them together into`
		`+ # the SQLALCHEMY_DATABASE_URI setting which is obeyed by Flask-SQLAlchemy.`
		`+ dburi = app.config['DATABASE_URI']`
		`+ if os.environ.get('DATABASE_PASSWORD'):`
		`+ parsed = urlparse.urlparse(dburi)`
		`+ netloc = '{}:{}@{}'.format(parsed.username,`
		`+ os.environ['DATABASE_PASSWORD'],`
		`+ parsed.hostname)`
		`+ if parsed.port:`
		`+ netloc += ':{}'.format(parsed.port)`
		`+ dburi = urlparse.urlunsplit(`
		`+ (parsed.scheme, netloc, parsed.path, parsed.query, parsed.fragment))`
		`+ if app.config['SHOW_DB_URI']:`
		`+ app.logger.debug('using DBURI: %s', dburi)`
		`+ app.config['SQLALCHEMY_DATABASE_URI'] = dburi`


		`# applicaiton factory http://flask.pocoo.org/docs/0.12/patterns/appfactories/`
		`@@ -46,8 +68,7 @@`
		`load_config(app)`
		`if app.config['PRODUCTION'] and app.secret_key == 'replace-me-with-something-random':`
		`raise Warning("You need to change the app.secret_key value for production")`
		`- if app.config['SHOW_DB_URI']:`
		`- app.logger.debug('using DBURI: %s' % app.config['SQLALCHEMY_DATABASE_URI'])`
		`+ populate_db_config(app)`
		`if app.config['AUTH_METHOD'] == 'OIDC':`
		`app.oidc = OpenIDConnect(app)`
		`# initialize db`
		`@@ -56,6 +77,7 @@`
		`init_logging(app)`
		`# register blueprints`
		`app.register_blueprint(api_v1, url_prefix="/api/v1.0")`
		`+ app.add_url_rule('/healthcheck', view_func=healthcheck)`
		`register_event_handlers(app)`
		`return app`

		`@@ -66,6 +88,19 @@`
		`return db`


		`+ def healthcheck():`
		`+ """`
		`+ Request handler for performing an application-level health check. This is`
		`+ not part of the published API, it is intended for use by OpenShift or other`
		`+ monitoring tools.`
		`+`
		`+ Returns a 200 response if the application is alive and able to serve requests.`
		`+ """`
		`+ result = db.session.execute('SELECT 1').scalar()`
		`+ assert result == 1`
		`+ return ('Health check OK', 200, [('Content-Type', 'text/plain')])`
		`+`
		`+`
		`def register_event_handlers(app):`
		`"""`
		`Register SQLAlchemy event handlers with the application's session factory.`

waiverdb/config.py

file modified

+2 -2

		`@@ -21,7 +21,7 @@`
		`fedmsg when new waivers are created.`
		`"""`
		`DEBUG = True`
		`- SQLALCHEMY_DATABASE_URI = 'sqlite://'`
		`+ DATABASE_URI = 'sqlite://'`
		`JOURNAL_LOGGING = False`
		`HOST = '0.0.0.0'`
		`PORT = 5004`
		`@@ -45,7 +45,7 @@`
		`class DevelopmentConfig(Config):`
		`SQLALCHEMY_TRACK_MODIFICATIONS = True`
		`TRAP_BAD_REQUEST_ERRORS = True`
		`- SQLALCHEMY_DATABASE_URI = 'sqlite:////var/tmp/waiverdb_db.sqlite'`
		`+ DATABASE_URI = 'sqlite:////var/tmp/waiverdb_db.sqlite'`
		`SHOW_DB_URI = True`
		`# The location of the client_secrets.json file used for API authentication`
		`OIDC_CLIENT_SECRETS = os.path.join(`

dcallagh commented 6 years ago

This PR has some fixes necessary to get the application running in OpenShift, and the first start towards deploying a Waiverdb test environment inside OpenShift and running tests against it.

dcallagh commented 6 years ago

Right now, everything in this PR is working as far as deploying the test environment from template manually using /usr/bin/oc.

However the Jenkins pieces are currently failing with this error which makes no sense to me:

ERROR: process returned an error;
{reference={}, err=error: unable to read certificate-authority /var/run/secrets/kubernetes.io/serviceaccount/ca.crt for  due to open /var/run/secrets/kubernetes.io/serviceaccount/ca.crt: no such file or directory, verb=process, cmd=oc process openshift/waiverdb-test-template.yaml -p TEST_ID=jenkins-waiverdb-dcallagh8-592 -p WAIVERDB_APP_VERSION=0.1.2.dev27-git.baec9a5 -f -o=json --server=https://open.paas.redhat.com/ --namespace=waiverdb-test --token=XXXXX , out=, status=1}

What I am not sure of right now is, why is it expecting /var/run/secrets/kubernetes.io/serviceaccount/ca.crt to exist and where should it be coming from? It seems to be a bug in the OpenShift Client Jenkins plugin but it needs more investigation.

mjia commented 6 years ago

That error may indicate that the OpenShift cluster isn't configured properly.

https://github.com/openshift/jenkins-client-plugin#configuring-an-openshift-cluster

dcallagh commented 6 years ago

Oh yeah I think you're right. When I filled in the cluster config into our Jenkins, I left the SSL certificate field blank, because I thought it would just use the system-wide trust store as normal. Seems like it doesn't do that though. A bit annoying... that's why we have a system-wide trust store, so that I don't have to configure the CA every single time. Oh well.

I have added the CA cert to the config and it seems to have got past that error now. Next problem is that openshift.process('-f', '<filename>'... does not work. I think we need to load the YAML template first and pass it in...

dcallagh commented 6 years ago

Okay, I have updated the last commit to 5a3dfec37eea6d27dd19b5640824e97da4246911 which gets a little further.

Have to wait for pods to be Ready, not just Running (they can start Running before the db is up, in which they will go into a crash loop until the db is reachable).

I got my selectors and stuff messed up, fixed those.

Now I am just running afoul of some Groovy scoping rules that I don't understand...

groovy.lang.MissingPropertyException: No such property: podReady for class: groovy.lang.Binding

😕

6 new commits added

Jenkinsfile: deploy and test the app inside OpenShift
add health checks for the web application
OpenShift template for producing a test environment
allow setting SECRET_KEY in the environment
allow setting DATABASE_PASSWORD in the environment
Dockerfile: include python-psycopg2

6 years ago

mprahl commented on line 13 of waiverdb/app.py 6 years ago

Could you prefix this with WAIVERDB, so that it's called WAIVERDB_SECRET_KEY?

mjia commented on line 41 of Jenkinsfile 6 years ago

As far as I understand in this example https://github.com/openshift/jenkins-client-plugin/blob/master/examples/coverage.groovy,
this should be written like this
pods.untilEach(3) {
def ready_cond = it.object().status.conditions.find { it.type == 'Ready' }
return ready_cond?.status == true
}

This may fix this error

groovy.lang.MissingPropertyException: No such property: podReady for class: groovy.lang.Binding

dcallagh commented on line 41 of Jenkinsfile 6 years ago

Nope I had that originally but you can't call .find in a closure because of JENKINS-26481... hence that @NonCPS business.

This pipeline stuff sounds promising in theory but is full of lots of nastiness once you start trying to use it for real...

alivigni commented 6 years ago

Is this output from the pipeline or an openshift log?

Can you provide some more details if they are available?

dcallagh commented on line 13 of waiverdb/app.py 6 years ago

Environment variables in OpenShift are scoped to each container so I don't think there is any potential for confusion if we just call this SECRET_KEY. Same as we don't have WAIVERDB_ prefixed on the database password, or krb5 keytab, etc.

mprahl commented on line 13 of waiverdb/app.py 6 years ago

Yes, but perhaps we shouldn't assume this will always be run in a container. I'm fairly certain you guys are deploying or have deployed a dev instance on a VM.

dcallagh commented 6 years ago

@alivigni the MissingPropertyException was from Jenkins. It was just a mistake in my Jenkinsfile. My Groovy fu is weak.

I was trying to pass my podReady function as if it were a first-class object like in Python. But in Groovy it's actually a method and you have to convert it to a closure using .& operator.

Anyway now my podReady method is still failing with com.cloudbees.groovy.cps.impl.CpsCallableInvocation although I had to hack it full of echo statements to even figure that out. I think this time I'm hitting JENKINS-31314... I've gotta say, this Pipeline stuff sounds nice in theory but is full of a lot of ridiculous gotchas...