feat: Build and push Schema Orchestrator

.gitlab-ci.yml

-image: golang:1.16
-
-stages:
-  - test
-  - docker
-  - deploy
-
-lint:
-  stage: test
-  script:
-    - make lint
-
-unit_tests:
-  stage: test
-  script:
-    - make dep
-    - make test
-
-docker:
-  image: docker:stable
-  tags:
-    - docker
-  stage: docker
-  only:
-    - develop
-    - main
-  before_script:
-    # Make sure docker uses Buildkit
-    - export DOCKER_BUILDKIT=1
-    # Log full docker build progress
-    - export BUILDKIT_PROGRESS=plain
-    # Enable experimental features (buildx)
-    - export DOCKER_CLI_EXPERIMENTAL=enabled
-    # Install docker buildx
-    - mkdir -p $HOME/.docker/cli-plugins
-    - wget -O $HOME/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.4.2/buildx-v0.4.2.linux-amd64
-    - chmod a+x $HOME/.docker/cli-plugins/docker-buildx
-    # Log in to our private image registry
-    - docker login -u $REGISTRY_USERNAME -p $REGISTRY_PASSWORD
-    - docker buildx install
-    - docker buildx create --use
-    # Install openssh and curl
-    - apk add openssh-client curl
-    - mkdir ~/.ssh/
-    - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    # Add our private ssh key
-    - ssh-add ~/.ssh/id_rsa
-    # Add UU gitlab to known hosts
-    - echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
-  script:
-    # Build and push image for linux/amd64 as well as linux/arm64
-    # Check if $CI_COMMIT_BRANCH is available, as it is not if this is a merge pipeline run
-    - if [[ ! -z $CI_COMMIT_BRANCH+x ]]; then DOCKER_TAG=$CI_COMMIT_BRANCH; else DOCKER_TAG=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME; fi
-    - docker buildx build --platform linux/amd64,linux/arm64 -t $REGISTRY_URL/$CI_PROJECT_NAME:$DOCKER_TAG . --push --ssh default
-
-deploy:
-  stage: deploy
-  only:
-    - develop
-    - main
-  script:
-    - apt-get install openssh-client curl -y >/dev/null
-    - mkdir ~/.ssh/
-    - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - ssh-add ~/.ssh/id_rsa
-    - echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
-    - ssh -fN -L 1234:science-vs260.science.uu.nl:22 sivan@up.science.uu.nl
-    # Copy kubernetes files over
-    - scp -r -o StrictHostKeyChecking=no -P 1234 -i ~/.ssh/id_rsa deployments/* root@localhost:/root/kubernetes/$CI_PROJECT_NAME
-    # Deploy all yml files
-    - ssh -p 1234 -i ~/.ssh/id_rsa root@localhost "kubectl apply -f kubernetes/$CI_PROJECT_NAME/"
-    # Set deployment image to the image we just made
-    - if [[ ! -z $CI_COMMIT_BRANCH+x ]]; then DOCKER_TAG=$CI_COMMIT_BRANCH; else DOCKER_TAG=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME; fi
-    # Restart the deployment if the image is not changed
-    - ssh -p 1234 -i ~/.ssh/id_rsa root@localhost "if kubectl set image deployment/$DEPLOYMENT_NAME $DEPLOYMENT_NAME=graphpolaris/$CI_PROJECT_NAME:$DOCKER_TAG | grep -q "image updated";
-      then
-        kubectl rollout restart deployment/$DEPLOYMENT_NAME;
-      fi"
-  dependencies: []
+# todo: Lint
+# todo: Unit tests
+
+variables:
+  CONTAINER_IMAGE: "graphpolaris/schema-orchestrator"
+  IMAGE_TAG_HELM_FILE: "helm/staging/schema-orchestrator/values.yaml"
+  IMAGE_TAG_HELM_FILE_VARIABLE: "base.image.tag"
+
+include:
+  - project: 'graphpolaris/pipelines'
+    ref: main
+    file: 'docker-build-push.yml'

Dockerfile

-# STAGE 1
-FROM golang:1.18
+FROM golang:1.18 as build
 
 ARG TARGETARCH
 ARG TARGETOS
 
 WORKDIR /app
+COPY . .
 
-# Use SSH instead of HTTPS
-# RUN echo "[url \"git@git.science.uu.nl:\"]\n\tinsteadOf = https://git.science.uu.nl/" >> /root/.gitconfig
+# Remove all lines starting with `replace` in go.mod
+RUN sed -i '/^replace/d' go.mod
 
-# Turn off strict host key checking
-# RUN mkdir /root/.ssh && echo "StrictHostKeyChecking no " > /root/.ssh/config
-
-# Copy go files into the image
-ADD dependencies /dependencies
-COPY go.mod ./
-COPY go.sum ./
-COPY cmd/ ./cmd/
-COPY internal/ ./internal/
-
-# Gather dependencies
 RUN go mod download
 
-# Compile for the target architecture and operating system
-# Add SSH mount as this operation requires access to private repos
 RUN GOARCH=${TARGETARCH} GOOS=${TARGETOS} CGO_ENABLED=0 go build -o ./main ./cmd/schema-orchestrator/
 
-# STAGE 2
-FROM busybox
+FROM alpine:3 as certs
 
-WORKDIR /app
+RUN apk update && \
+    apk --no-cache add ca-certificates
 
-# Copy the built binary into this image
-COPY --from=0 /app/main ./
+FROM alpine:3
+
+COPY --from=certs /etc/ssl/certs /etc/ssl/certs
+
+WORKDIR /app
+COPY --from=build /app/main ./
 
-# Run the binary
-CMD ./main
+CMD ["./main"]
\ No newline at end of file

go.mod

@@ -3,8 +3,8 @@ module schema-orchestrator
 go 1.18
 
 require (
-	git.science.uu.nl/graphpolaris/broker v0.0.0-20210913145737-76d174f4367a
-	git.science.uu.nl/graphpolaris/keyvaluestore v0.0.0-20211015140849-feaa365e3730
+	git.science.uu.nl/graphpolaris/broker v0.0.0-20230809123708-7d1c1acf3e91
+	git.science.uu.nl/graphpolaris/keyvaluestore v0.0.0-20230829154955-1d33a3308a57
 	git.science.uu.nl/graphpolaris/objectstore v0.0.0-20210913150113-977062fb8a3c
 	github.com/arangodb/go-driver v1.2.1
 	github.com/go-chi/cors v1.2.1
@@ -16,33 +16,22 @@ require (
 require (
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-redis/redis/v8 v8.8.2 // indirect
 	github.com/golang/protobuf v1.4.3 // indirect
-	github.com/google/uuid v1.1.2 // indirect
-	github.com/json-iterator/go v1.1.10 // indirect
-	github.com/klauspost/cpuid v1.3.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.18 // indirect
-	github.com/minio/md5-simd v1.1.0 // indirect
-	github.com/minio/minio-go/v7 v7.0.10 // indirect
-	github.com/minio/sha256-simd v0.1.1 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rs/xid v1.4.0 // indirect
 	github.com/streadway/amqp v1.0.0 // indirect
 	github.com/thijsheijden/alice v0.1.21-0.20220206104614-72a8777edcc7 // indirect
 	go.opentelemetry.io/otel v0.19.0 // indirect
 	go.opentelemetry.io/otel/metric v0.19.0 // indirect
 	go.opentelemetry.io/otel/trace v0.19.0 // indirect
-	golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e // indirect
 	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
 	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
-	gopkg.in/ini.v1 v1.57.0 // indirect
 )
 
 replace git.science.uu.nl/graphpolaris/broker => ../../dependencies/broker

go.sum

 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+git.science.uu.nl/graphpolaris/broker v0.0.0-20230809123708-7d1c1acf3e91 h1:Y0nz7IRn74AQmCMPNH0Njn8MUMb/NwYex6lwR6Zz0wc=
+git.science.uu.nl/graphpolaris/broker v0.0.0-20230809123708-7d1c1acf3e91/go.mod h1:+fmTCX+w0lRhFX1JVP3WI5OgQeOl8wbsAoA6g7J2f6g=
+git.science.uu.nl/graphpolaris/keyvaluestore v0.0.0-20230829154955-1d33a3308a57 h1:ky8mvnx1vi1mWpsAIZ/JzLq23hz625L4aiEQk/GiC1E=
+git.science.uu.nl/graphpolaris/keyvaluestore v0.0.0-20230829154955-1d33a3308a57/go.mod h1:r62eFpVrFJldxyr0AC2ww7pDcEVUpdAh4X9YdsGEzss=
+git.science.uu.nl/graphpolaris/objectstore v0.0.0-20210913150113-977062fb8a3c/go.mod h1:JTrEtLWqP0uUvOdS8l5+nINwHSoY1KwRYL1i86D/WCU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/arangodb/go-driver v1.2.1 h1:HREDHhDmzdIWxHmfkfTESbYUnRjESjPh4WUuXq7FZa8=
@@ -62,18 +67,13 @@ github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
 github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -85,19 +85,13 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
 github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
-github.com/minio/minio-go/v7 v7.0.10 h1:1oUKe4EOPUEhw2qnPQaPsJ0lmVTYLFu03SiItauXs94=
 github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
-github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -118,15 +112,12 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
 github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc=
 github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
 github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo=
 github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
@@ -153,7 +144,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e h1:1SzTfNOXwIS2oWiMF+6qu0OUDKb0dauo6MoDUQyu+yU=
 golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -254,7 +244,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
 gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=