CircleCI: dockerhub builds (#583)

Author: rafie

.circleci/config.yml

@@ -106,6 +106,27 @@ commands:
       - store_artifacts:
           path: test/logs
 
+  dockerhub-build-steps:
+    parameters:
+      build_args:
+        type: string
+    steps:
+      - setup-executor
+      - checkout
+      - run:
+          name: Setup
+          command: |
+            git submodule update --init --recursive
+            ./opt/readies/bin/getpy3
+      - run:
+          name: Build for platform
+          command: |
+            docker login -u redisfab -p $DOCKER_REDISFAB_PWD
+            cd opt/build/docker
+            make build publish $(./version-params) CPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <<parameters.build_args>>
+            make build publish $(./version-params) GPU=1 OSNICK=bionic X64=1 REPO=redislabs VERBOSE=1 <<parameters.build_args>>
+          no_output_timeout: 40m
+
   deploy-steps:
     parameters:
       from:
@@ -131,9 +152,9 @@ jobs:
           command: |
             make -C opt lint
 
-  build-debian:
+  build-and-test:
     docker:
-      - image: redisfab/rmbuilder:6.0.5-x64-buster
+      - image: redisfab/rmbuilder:6.0.9-x64-buster
     steps:
       - build-steps:
           platform: debian
@@ -150,7 +171,7 @@ jobs:
 
   coverage:
     docker:
-      - image: redisfab/rmbuilder:6.0.5-x64-buster
+      - image: redisfab/rmbuilder:6.0.9-x64-buster
     steps:
       - checkout
       - run:
@@ -178,8 +199,12 @@ jobs:
           no_output_timeout: 30m
 
   valgrind:
+    parameters:
+      test_args:
+        type: string
+        default: "CLUSTER=0 AOF=0"
     docker:
-      - image: redisfab/rmbuilder:6.0.5-x64-buster
+      - image: redisfab/rmbuilder:6.0.9-x64-buster
     steps:
       - checkout
       - run:
@@ -202,63 +227,7 @@ jobs:
       - run:
           name: Test with valgrind
           command: |
-            make -C opt test VALGRIND=1 CLUSTER=0 AOF=0
-          no_output_timeout: 120m
-
-  valgrind-cluster:
-    docker:
-      - image: redisfab/rmbuilder:6.0.5-x64-buster
-    steps:
-      - checkout
-      - run:
-          name: Submodule checkout
-          command: git submodule update --init --recursive
-      - restore_cache:
-          keys:
-            - build-dependencies-{{ checksum "get_deps.sh" }}
-          # If no exact match is found will get dependencies from source
-      - setup-automation
-      - run:
-          name: Install dependencies
-          command: |
-            ./opt/readies/bin/getredis -v 6 --valgrind --force
-            ./get_deps.sh cpu
-      - run:
-          name: Build for valgrind with cluster
-          command: |
-            make -C opt all VALGRIND=1 SHOW=1
-      - run:
-          name: Test with valgrind and cluster
-          command: |
-            make -C opt test VALGRIND=1 GEN=0 AOF=0
-          no_output_timeout: 120m
-
-  valgrind-AOF:
-    docker:
-      - image: redisfab/rmbuilder:6.0.5-x64-buster
-    steps:
-      - checkout
-      - run:
-          name: Submodule checkout
-          command: git submodule update --init --recursive
-      - restore_cache:
-          keys:
-            - build-dependencies-{{ checksum "get_deps.sh" }}
-          # If no exact match is found will get dependencies from source
-      - setup-automation
-      - run:
-          name: Install dependencies
-          command: |
-            ./opt/readies/bin/getredis -v 6 --valgrind --force
-            ./get_deps.sh cpu
-      - run:
-          name: Build for valgrind with AOF
-          command: |
-            make -C opt all VALGRIND=1 SHOW=1
-      - run:
-          name: Test with valgrind and AOF
-          command: |
-            make -C opt test VALGRIND=1 GEN=0 CLUSTER=0
+            make -C opt test VALGRIND=1 <<parameters.test_args>>
           no_output_timeout: 120m
 
   build-macos:
@@ -325,12 +294,22 @@ jobs:
       - store_artifacts:
           path: test/log
 
+  dockerhub-build:
+    parameters:
+      build_args:
+        type: string
+    docker:
+      - image: debian:buster
+    steps:
+      - dockerhub-build-steps:
+          build_args: <<parameters.build_args>>
+
   deploy-artifacts:
     parameters:
       location:
         type: string
     docker:
-      - image: redisfab/rmbuilder:6.0.5-x64-buster
+      - image: redisfab/rmbuilder:6.0.9-x64-buster
     steps:
       - attach_workspace:
           at: workspace
@@ -344,7 +323,7 @@ jobs:
 
   deploy-snapshot:
     docker:
-      - image: redisfab/rmbuilder:6.0.5-x64-buster
+      - image: redisfab/rmbuilder:6.0.9-x64-buster
     steps:
       - attach_workspace:
           at: workspace
@@ -362,7 +341,7 @@ jobs:
 
   deploy-release:
     docker:
-      - image: redisfab/rmbuilder:6.0.5-x64-buster
+      - image: redisfab/rmbuilder:6.0.9-x64-buster
     steps:
       - attach_workspace:
           at: workspace
@@ -422,29 +401,33 @@ on-integ-and-version-tags: &on-integ-and-version-tags
       only: /^v[0-9].*/
 
 
-platform-build-defs: &platform-build-defs
+after-linter: &after-linter
   requires:
-    - build-debian
-  <<: *on-integ-and-version-tags
-  # <<: *on-any-branch # debugging
+    - lint
+
+after-build-and-test: &after-build-and-test
+  requires:
+    - build-and-test
 
 after-platform-builds: &after-platform-builds
   requires:
     # - build-centos7
     - build-bionic
     - build-xenial
 
-after-linter: &after-linter
-  requires:
-    - lint
+platform-build-defs: &platform-build-defs
+  <<: *after-build-and-test
+  <<: *on-integ-and-version-tags
+  # <<: *on-any-branch # debugging
+
 
 workflows:
   version: 2
   build_and_package:
     jobs:
       - lint:
           <<: *on-any-branch
-      - build-debian:
+      - build-and-test:
           <<: *on-any-branch
           <<: *after-linter
       - platform-build:
@@ -460,15 +443,21 @@ workflows:
           platform: xenial
           <<: *platform-build-defs
       - coverage:
+          context: common
           <<: *on-any-branch
           <<: *after-linter
       - valgrind:
+          test_args: CLUSTER=0 AOF=0
           <<: *on-any-branch
           <<: *after-linter
-      - valgrind-cluster:
+      - valgrind:
+          name: valgrind-cluster
+          test_args: GEN=0 AOF=0
           <<: *on-integ-branch
           <<: *after-linter
-      - valgrind-AOF:
+      - valgrind:
+          name: valgrind-aof
+          test_args: GEN=0 CLUSTER=0
           <<: *on-integ-branch
           <<: *after-linter
       - build-and-test-gpu:
@@ -478,12 +467,27 @@ workflows:
           <<: *never # temporarily disabled
           # <<: *on-version-tags
       - build-multiarch-docker:
+          context: common
           <<: *never # temporarily disabled
           # <<: *on-version-tags
       - deploy-snapshot:
+          context: common
           <<: *after-platform-builds
           <<: *on-integ-branch
+      - dockerhub-build:
+          name: dockerhub-edge
+          context: common
+          build_args: EDGE=1
+          <<: *after-build-and-test
+          <<: *on-master
       - deploy-release:
+          context: common
+          <<: *after-platform-builds
+          <<: *on-version-tags
+      - dockerhub-build:
+          name: dockerhub-release
+          context: common
+          build_args: LATEST=1
           <<: *after-platform-builds
           <<: *on-version-tags
 

Dockerfile

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.5
+ARG REDIS_VER=6.0.9
 
 # OSNICK=bionic|stretch|buster
 ARG OSNICK=buster
@@ -31,7 +31,7 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./tests/flow/test_requirements.txt tests/flow/
 
-RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
+RUN FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS=""

Dockerfile.arm

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-cpu-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.5
+ARG REDIS_VER=6.0.9
 
 # OSNICK=bionic|stretch|buster
 ARG OSNICK=buster

Dockerfile.gpu

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}
 
-ARG REDIS_VER=6.0.5
+ARG REDIS_VER=6.0.9
 
 # OSNICK=bionic|centos7|centos6
 ARG OSNICK=bionic
@@ -37,7 +37,7 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./tests/flow/test_requirements.txt tests/flow/
 
-RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
+RUN FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS=""

Dockerfile.gpu-test

@@ -1,6 +1,6 @@
 # BUILD redisfab/redisai:${VERSION}-gpu-${ARCH}-${OSNICK}-test
 
-ARG REDIS_VER=6.0.5
+ARG REDIS_VER=6.0.9
 
 # OSNICK=bionic|centos7|centos6
 ARG OSNICK=bionic
@@ -30,12 +30,10 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./tests/flow/test_requirements.txt tests/flow/
 
-RUN PIP=1 VENV=1 FORCE=1 ./opt/readies/bin/getpy3
+RUN VENV=venv FORCE=1 ./opt/readies/bin/getpy3
 
 RUN set -e ;\
-	python3 -m virtualenv venv --system-site-packages ;\
 	. venv/bin/activate ;\
-	python -m pip install pip==19.3.1 ;\
 	./opt/system-setup.py
 
 ARG DEPS_ARGS=""

Dockerfile.jetson

@@ -40,7 +40,7 @@ COPY --from=redis /usr/local/ /usr/local/
 COPY ./opt/ opt/
 COPY ./tests/flow/test_requirements.txt tests/flow/
 
-RUN PIP=1 FORCE=1 ./opt/readies/bin/getpy3
+RUN FORCE=1 ./opt/readies/bin/getpy3
 RUN ./opt/system-setup.py
 
 ARG DEPS_ARGS="GPU=1 JETSON=1 WITH_PT=1 WITH_TF=1 WITH_TFLITE=0 WITH_ORT=0"

opt/build/docker/Makefile

@@ -6,9 +6,16 @@ include $(ROOT)/opt/readies/mk/main
 # OSNICK=buster|stretch|bionic|xenial|centos7
 OSNICK ?= buster
 
+ifeq ($(REPO),redislabs)
+SIMPLE ?= 1
+else
 REPO=redisfab
+endif
+
 STEM=$(REPO)/redisai
 
+LATEST_BRANCH=1.2
+
 REDIS_VER:=6.0.9
 
 ART_DIR=$(ROOT)/bin/artifacts