Multi Language CI (#11674)

* feat: add multi language test

* fix: replace dockerhub

.github/workflows/pr-e2e-test.yml

@@ -6,10 +6,11 @@ on:
     workflows: ["PR-CI"]
     types:
       - completed
-  
+
 env:
   DOCKER_REPO: wuyfeedocker/nacos-ci
-  ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+  DOCKER_REPO_B: wuyfeehub/nacos-ci
+  TEST_REPO_NAME: nacos-group/nacos-e2e
 
 jobs:
   docker:
@@ -22,6 +23,8 @@ jobs:
       matrix:
         base-image: ["ubuntu"]
         java-version: ["8"]
+    outputs:
+      version-json: ${{ steps.show_versions.outputs.version-json }}
     steps:
       - name: 'Download artifact'
         uses: actions/github-script@v3.1.0
@@ -33,7 +36,7 @@ jobs:
                run_id: ${{ github.event.workflow_run.id }},
             });
             var matchArtifactNacos = artifacts.data.artifacts.filter((artifact) => {
-              return artifact.name == "nacos"
+               return artifact.name == "nacos"
             })[0];
             var download = await github.actions.downloadArtifact({
                owner: context.repo.owner,
@@ -47,19 +50,12 @@ jobs:
           unzip nacos.zip
           mkdir nacos
           cp -r nacos-* nacos/
-          ls
       - uses: actions/checkout@v3
         with:
           repository: nacos-group/nacos-e2e.git
           ref: main
           path: nacos-e2e
-      - name: docker-login
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and save docker images
+      - name: Generate image tag
         id: build-images
         run: |
           mv nacos-server-*.tar.gz nacos-e2e/cicd/build
@@ -68,43 +64,44 @@ jobs:
           mkdir versionlist
           touch versionlist/"${version}-`echo ${{ matrix.java-version }} | sed -e "s/:/-/g"`"
           ls versionlist/
-          tag=${version}-$(echo ${{ matrix.java-version }} | sed -e "s/:/-/g")
-          echo $tag
-          docker build --no-cache -f Dockerfile -t ${DOCKER_REPO}:${tag} .
-          docker push ${DOCKER_REPO}:${tag}
-      - uses: actions/upload-artifact@v3
-        name: Upload distribution tar
+          echo TAG=${version}-$(echo ${{ matrix.java-version }} | sed -e "s/:/-/g") >> $GITHUB_ENV
+      - name: docker-login-1
+        uses: docker/login-action@v2
         with:
-          name: versionlist
-          path: nacos-e2e/cicd/build/versionlist/*
-
-
-  list-version:
-    if: always()
-    name: List version
-    needs: [docker]
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    outputs:
-      version-json: ${{ steps.show_versions.outputs.version-json }}
-    steps:
-      - uses: actions/download-artifact@v3
-        name: Download versionlist
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: save docker_1 images
+        run: |
+          mkdir build_backup
+          cp -rf nacos-e2e/cicd/build/* ./build_backup/
+          cd nacos-e2e/cicd/build
+          docker build --no-cache -f Dockerfile -t ${DOCKER_REPO}:${{ env.TAG }} .
+          docker push ${DOCKER_REPO}:${{ env.TAG }}
+      - name: docker-login-2
+        uses: docker/login-action@v2
         with:
-          name: versionlist
-          path: versionlist
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.DOCKERHUB_USER_B }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_B }}
+      - name: save docker_2 images
+        run: |
+          rm -rf nacos-e2e/cicd/build/*
+          mv ./build_backup/* nacos-e2e/cicd/build/
+          cd nacos-e2e/cicd/build
+          docker build --no-cache -f Dockerfile -t ${DOCKER_REPO_B}:${{ env.TAG }} .
+          docker push ${DOCKER_REPO_B}:${{ env.TAG }}
       - name: Show versions
         id: show_versions
-        run: | 
-          a=(`ls versionlist`)
+        run: |
+          a=(`ls nacos-e2e/cicd/build/versionlist`)
           printf '%s\n' "${a[@]}" | jq -R . | jq -s .
           echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT
-  
-  
+
   deploy:
     if: ${{ success() }}
     name: Deploy nacos
-    needs: [list-version,docker]
+    needs: [docker]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
@@ -112,99 +109,392 @@ jobs:
       DATABASE: mysql
       NODE_PORT: 30000
     strategy:
+      fail-fast: false
       matrix:
         mode: ["cluster","standalone"]
-        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
     steps:
       - name: set nodeport
         run: |
-            echo "::set-env name=NODE_PORT::$(expr $(expr $(expr $(expr 1 * $(expr ${{ github.run_number }} - 1)) + ${{ strategy.job-index }}) % 30000) + 30000)"
+          echo "NODE_PORT=$(expr $(expr $(expr $(expr ${{ strategy.job-index }} + 1) * ${{ github.run_number }}) % 30000) + 30000)" >> $GITHUB_ENV
       - name: set cluster params
         if: ${{ matrix.mode == 'standalone' }}
         run: |
-          echo "::set-env name=REPLICA_COUNT::1"
-          echo "::set-env name=DATABASE::embedded"
-      - uses: apache/rocketmq-test-tool@136205caa6e97f81744e30adea114dd2f09cc55e
+          echo "REPLICA_COUNT=1" >> $GITHUB_ENV
+          echo "DATABASE=embedded" >> $GITHUB_ENV
+      - name: allocate docker repo
+        run: |
+          if [[ $(expr $(expr ${{ github.run_id }} + ${{ strategy.job-index }} ) % 2 ) -eq 1 ]]; then
+            echo "DOCKER_REPO_ACTUAL=${{ env.DOCKER_REPO }}" >> $GITHUB_ENV
+          else
+            echo "DOCKER_REPO_ACTUAL=${{ env.DOCKER_REPO_B }}" >> $GITHUB_ENV
+          fi
+      - uses: apache/rocketmq-test-tool@java-dev
         name: Deploy nacos
         with:
-          action: "deploy"
-          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
-          test-version: "${{ matrix.version }}"
-          chart-git: "https://ghproxy.com/https://github.com/nacos-group/nacos-e2e.git"
-          chart-branch: "main"
-          chart-path: "./cicd/helm"
-          job-id: ${{ strategy.job-index }}
-          helm-values: |
+          yamlString: |
+            action: deploy
             namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
-            global:
-              mode: ${{ matrix.mode }}
-            nacos:
-              replicaCount: ${{ env.REPLICA_COUNT }}
-              image: 
-                repository: ${{ env.DOCKER_REPO }}
-                tag: ${{ matrix.version }}
-              storage:
-                type: ${{ env.DATABASE }}
-                db:
-                  port: 3306
-                  username: nacos
-                  password: nacos
-                  param: characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false
-            service:
-              nodePort: ${{ env.NODE_PORT }}
-              type: ClusterIP
-  e2e-test:
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            waitTimes: 2000
+            velaAppDescription: nacos-${{ env.GITHUB_WORKFLOW }}-${{ github.run_id }}@${{ matrix.version }}
+            repoName: nacos
+            helm:
+              chart: ./cicd/helm
+              git:
+                branch: main
+              repoType: git
+              retries: 3
+              url: https://ghproxy.com/https://github.com/nacos-group/nacos-e2e.git
+              values:
+                namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+                global:
+                  mode: ${{ matrix.mode }}
+                nacos:
+                  replicaCount: ${{ env.REPLICA_COUNT }}
+                  image:
+                    repository: ${{ env.DOCKER_REPO_ACTUAL }}
+                    tag: ${{ matrix.version }}
+                  storage:
+                    type: ${{ env.DATABASE }}
+                    db:
+                      port: 3306
+                      username: nacos
+                      password: nacos
+                      param: characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false
+                service:
+                  nodePort: ${{ env.NODE_PORT }}
+                  type: ClusterIP
+  e2e-java-test:
     if: ${{ success() }}
-    name: E2E Test
-    needs: [list-version, deploy]
+    name: Java e2e Test
+    needs: [docker, deploy]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
+      fail-fast: false
       matrix:
         mode: ["cluster","standalone"]
-        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
     steps:
-      - uses: apache/rocketmq-test-tool@136205caa6e97f81744e30adea114dd2f09cc55e
-        name: e2e test
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: java e2e test
         with:
-          action: "test"
-          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
-          test-version: "${{ matrix.version }}"
-          test-code-git: "https://github.com/nacos-group/nacos-e2e.git"
-          test-code-branch: "main"
-          test-code-path: "java/nacos-2X"
-          test-cmd: 'mvn clean test -B'
-          job-id: ${{ strategy.job-index }}
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v3
-        if: always() # always run even if the previous step fails
-        with:
-          report_paths: '**/test_report/TEST-*.xml'
-          annotate_only: true
-          include_passed: true
-          detailed_summary: true
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: java/nacos-2X
+              CMD: mvn clean test -B
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+
       - uses: actions/upload-artifact@v3
         if: always()
         name: Upload test log
         with:
-          name: testlog.txt ${{ matrix.mode }}
+          name: testlog-${{ matrix.mode }}-java.txt
           path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+
+  e2e-go-test:
+    if: ${{ success() }}
+    name: GO E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: go e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: golang
+              CMD: |
+                cd /root/code/golang && go mod init nacos_go_test && go mod tidy
+                gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./nacosgotest
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-go.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+  e2e-cpp-test:
+    if: ${{ success() }}
+    name: Cpp E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: cpp e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: cpp
+              CMD: |
+                cd /root/code/cpp && make install
+                echo "export LD_LIBRARY_PATH=/usr/local/lib" >> ~/.bashrc  && source ~/.bashrc
+                cd /root/code/cpp/nacoscpptest
+                g++ nacos_test.cpp -o nacos_test -lgtest -lpthread -I/usr/local/include/nacos/ -L/usr/local/lib/  -lnacos-cli
+                chmod 777 nacos_test && ./nacos_test --gtest_output="xml:../target/surefire-reports/TEST-gtestresults.xml"
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-cpp.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+  e2e-csharp-test:
+    if: ${{ success() }}
+    name: Csharp E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: csharp e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: csharp
+              CMD: |
+                rpm -Uvh https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm
+                yum -y install dotnet-sdk-3.1 && yum -y install aspnetcore-runtime-7.0
+                cd /root/code/csharp/nacos-csharp-sdk-test && dotnet restore
+                dotnet test --logger:"junit;LogFilePath=../target/surefire-reports/TEST-result.xml"
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-csharp.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+  e2e-nodejs-test:
+    if: ${{ success() }}
+    name: Nodejs E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: nodejs e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: nodejs
+              CMD: |
+                cd /root/code/nodejs/nacosnodejstest && npm install
+                mocha test --reporter mocha-junit-reporter --reporter-options mochaFile=../target/surefire-reports/TEST-report.xml
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-nodejs.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+  e2e-python-test:
+    if: ${{ success() }}
+    name: Python E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: python e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: python
+              CMD: |
+                cd /root/code/python 
+                pip3 install -r requirements.txt
+                source ~/.bashrc
+                cd nacospythontest && pytest --junitxml ../target/surefire-reports/TEST-report.xml test/*_test.py --log-cli-level=DEBUG
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-python.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
 
   clean:
     if: always()
     name: Clean
-    needs: [list-version, e2e-test]
+    needs: [docker, e2e-java-test, e2e-go-test, e2e-cpp-test, e2e-csharp-test, e2e-nodejs-test, e2e-python-test]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
+      fail-fast: false
       matrix:
         mode: ["cluster","standalone"]
-        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
     steps:
-      - uses: apache/rocketmq-test-tool@136205caa6e97f81744e30adea114dd2f09cc55e
+      - uses: apache/rocketmq-test-tool@java-dev
         name: clean
         with:
-          action: "clean"
-          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
-          test-version: "${{ matrix.version }}"
-          job-id: ${{ strategy.job-index }}
+          yamlString: |
+            action: clean
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}

.github/workflows/push-ci.yaml

@@ -5,15 +5,15 @@ on:
     branches: [master, develop, v1.x-develop, v1.X]
 
 permissions:
-  contents: read # to fetch code (actions/checkout)
+  contents: read
 
 concurrency:
   group: nacos-${{ github.ref }}
 
 env:
   DOCKER_REPO: wuyfeedocker/nacos-ci
-  ACTIONS_ALLOW_UNSECURE_COMMANDS: true
-
+  DOCKER_REPO_B: wuyfeehub/nacos-ci
+  TEST_REPO_NAME: nacos-group/nacos-e2e
 
 jobs:
   dist-tar:
@@ -50,10 +50,15 @@ jobs:
     needs: [dist-tar]
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    env:
+      DOCKERHUB_USER_ACTUAL: ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_TOKEN_ACTUAL: ${{ secrets.DOCKERHUB_TOKEN }}
     strategy:
       matrix:
         base-image: ["centos"]
         java-version: ["8"]
+    outputs:
+      version-json: ${{ steps.show_versions.outputs.version-json }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -65,13 +70,7 @@ jobs:
         with:
           name: nacos
           path: ./
-      - name: docker-login
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and save docker images
+      - name: Generate image tag
         id: build-images
         run: |
           mv nacos-server-*.tar.gz nacos-e2e/cicd/build/
@@ -80,42 +79,44 @@ jobs:
           mkdir versionlist
           touch versionlist/"${version}-`echo ${{ matrix.java-version }} | sed -e "s/:/-/g"`"
           ls versionlist/
-          tag=${version}-$(echo ${{ matrix.java-version }} | sed -e "s/:/-/g")
-          echo $tag
-          docker build --no-cache -f Dockerfile -t ${DOCKER_REPO}:${tag} .
-          docker push ${DOCKER_REPO}:${tag}
-      - uses: actions/upload-artifact@v3
-        name: Upload distribution tar
+          echo TAG=${version}-$(echo ${{ matrix.java-version }} | sed -e "s/:/-/g") >> $GITHUB_ENV
+      - name: docker-login-1
+        uses: docker/login-action@v2
         with:
-          name: versionlist
-          path: nacos-e2e/cicd/build/versionlist/*
-          
-          
-  list-version:
-    if: always()
-    name: List version
-    needs: [docker]
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    outputs:
-      version-json: ${{ steps.show_versions.outputs.version-json }}
-    steps:
-      - uses: actions/download-artifact@v3
-        name: Download versionlist
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: save docker_1 images
+        run: |
+          mkdir build_backup
+          cp -rf nacos-e2e/cicd/build/* ./build_backup/
+          cd nacos-e2e/cicd/build
+          docker build --no-cache -f Dockerfile -t ${DOCKER_REPO}:${{ env.TAG }} .
+          docker push ${DOCKER_REPO}:${{ env.TAG }}
+      - name: docker-login-2
+        uses: docker/login-action@v2
         with:
-          name: versionlist
-          path: versionlist
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.DOCKERHUB_USER_B }}
+          password: ${{ secrets.DOCKERHUB_TOKEN_B }}
+      - name: save docker_2 images
+        run: |
+          rm -rf nacos-e2e/cicd/build/*
+          mv ./build_backup/*  nacos-e2e/cicd/build/
+          cd nacos-e2e/cicd/build
+          docker build --no-cache -f Dockerfile -t ${DOCKER_REPO_B}:${{ env.TAG }} .
+          docker push ${DOCKER_REPO_B}:${{ env.TAG }}
       - name: Show versions
         id: show_versions
         run: | 
-          a=(`ls versionlist`)
+          a=(`ls nacos-e2e/cicd/build/versionlist`)
           printf '%s\n' "${a[@]}" | jq -R . | jq -s .
           echo version-json=`printf '%s\n' "${a[@]}" | jq -R . | jq -s .` >> $GITHUB_OUTPUT
-          
+
   deploy:
     if: ${{ success() }}
     name: Deploy nacos
-    needs: [list-version,docker]
+    needs: [docker]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     env:
@@ -123,99 +124,390 @@ jobs:
       DATABASE: mysql
       NODE_PORT: 30000
     strategy:
+      fail-fast: false
       matrix:
         mode: ["cluster","standalone"]
-        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
     steps:
       - name: set nodeport
         run: |
-            echo "::set-env name=NODE_PORT::$(expr $(expr $(expr $(expr 1 * $(expr ${{ github.run_number }} - 1)) + ${{ strategy.job-index }}) % 30000) + 30000)"
+          echo "NODE_PORT=$(expr $(expr $(expr $(expr ${{ strategy.job-index }} + 1) * ${{ github.run_number }}) % 30000) + 30000)" >> $GITHUB_ENV
       - name: set cluster params
         if: ${{ matrix.mode == 'standalone' }}
         run: |
-          echo "::set-env name=REPLICA_COUNT::1"
-          echo "::set-env name=DATABASE::embedded"
-      - uses: apache/rocketmq-test-tool@136205caa6e97f81744e30adea114dd2f09cc55e
+          echo "REPLICA_COUNT=1" >> $GITHUB_ENV
+          echo "DATABASE=embedded" >> $GITHUB_ENV
+      - name: allocate docker repo
+        run: |
+          if [[ $(expr $(expr ${{ github.run_id }} + ${{ strategy.job-index }} ) % 2 ) -eq 1 ]]; then
+            echo "DOCKER_REPO_ACTUAL=${{ env.DOCKER_REPO }}" >> $GITHUB_ENV
+          else
+            echo "DOCKER_REPO_ACTUAL=${{ env.DOCKER_REPO_B }}" >> $GITHUB_ENV
+          fi
+      - uses: apache/rocketmq-test-tool@java-dev
         name: Deploy nacos
         with:
-          action: "deploy"
-          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
-          test-version: "${{ matrix.version }}"
-          chart-git: "https://ghproxy.com/https://github.com/nacos-group/nacos-e2e.git"
-          chart-branch: "main"
-          chart-path: "./cicd/helm"
-          job-id: ${{ strategy.job-index }}
-          helm-values: |
+          yamlString: |
+            action: deploy
             namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
-            global:
-              mode: ${{ matrix.mode }}
-            nacos:
-              replicaCount: ${{ env.REPLICA_COUNT }}
-              image: 
-                repository: ${{ env.DOCKER_REPO }}
-                tag: ${{ matrix.version }}
-              storage:
-                type: ${{ env.DATABASE }}
-                db:
-                  port: 3306
-                  username: nacos
-                  password: nacos
-                  param: characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false
-            service:
-              nodePort: ${{ env.NODE_PORT }}
-              type: ClusterIP
-  e2e-test:
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            waitTimes: 2000
+            velaAppDescription: nacos-${{ env.GITHUB_WORKFLOW }}-${{ github.run_id }}@${{ matrix.version }}
+            repoName: nacos
+            helm:
+              chart: ./cicd/helm
+              git:
+                branch: main
+              repoType: git
+              retries: 3
+              url: https://ghproxy.com/https://github.com/nacos-group/nacos-e2e.git
+              values:
+                namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+                global:
+                  mode: ${{ matrix.mode }}
+                nacos:
+                  replicaCount: ${{ env.REPLICA_COUNT }}
+                  image:
+                    repository: ${{ env.DOCKER_REPO_ACTUAL }}
+                    tag: ${{ matrix.version }}
+                  storage:
+                    type: ${{ env.DATABASE }}
+                    db:
+                      port: 3306
+                      username: nacos
+                      password: nacos
+                      param: characterEncoding=utf8&connectTimeout=1000&socketTimeout=3000&autoReconnect=true&useSSL=false
+                service:
+                  nodePort: ${{ env.NODE_PORT }}
+                  type: ClusterIP
+  e2e-java-test:
     if: ${{ success() }}
-    name: E2E Test
-    needs: [list-version, deploy]
+    name: Java e2e Test
+    needs: [docker, deploy]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
+      fail-fast: false
       matrix:
         mode: ["cluster","standalone"]
-        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
     steps:
-      - uses: apache/rocketmq-test-tool@136205caa6e97f81744e30adea114dd2f09cc55e
-        name: e2e test
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: java e2e test
         with:
-          action: "test"
-          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
-          test-version: "${{ matrix.version }}"
-          test-code-git: "https://github.com/nacos-group/nacos-e2e.git"
-          test-code-branch: "main"
-          test-code-path: "java/nacos-2X"
-          test-cmd: 'mvn clean test -B'
-          job-id: ${{ strategy.job-index }}
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v3
-        if: always() # always run even if the previous step fails
-        with:
-          report_paths: '**/test_report/TEST-*.xml'
-          annotate_only: true
-          include_passed: true
-          detailed_summary: true
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: java/nacos-2X
+              CMD: mvn clean test -B
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+
       - uses: actions/upload-artifact@v3
         if: always()
         name: Upload test log
         with:
-          name: testlog.txt ${{ matrix.mode }}
+          name: testlog-${{ matrix.mode }}-java.txt
           path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+
+  e2e-go-test:
+    if: ${{ success() }}
+    name: GO E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: go e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: golang
+              CMD: |
+                cd /root/code/golang && go mod init nacos_go_test && go mod tidy
+                gotestsum --junitfile ./target/surefire-reports/TEST-report.xml ./nacosgotest
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-go.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+  e2e-cpp-test:
+    if: ${{ success() }}
+    name: Cpp E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: cpp e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: cpp
+              CMD: |
+                cd /root/code/cpp && make install
+                echo "export LD_LIBRARY_PATH=/usr/local/lib" >> ~/.bashrc  && source ~/.bashrc
+                cd /root/code/cpp/nacoscpptest
+                g++ nacos_test.cpp -o nacos_test -lgtest -lpthread -I/usr/local/include/nacos/ -L/usr/local/lib/  -lnacos-cli
+                chmod 777 nacos_test && ./nacos_test --gtest_output="xml:../target/surefire-reports/TEST-gtestresults.xml"
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-cpp.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+  e2e-csharp-test:
+    if: ${{ success() }}
+    name: Csharp E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: csharp e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: csharp
+              CMD: |
+                rpm -Uvh https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm
+                yum -y install dotnet-sdk-3.1 && yum -y install aspnetcore-runtime-7.0
+                cd /root/code/csharp/nacos-csharp-sdk-test && dotnet restore
+                dotnet test --logger:"junit;LogFilePath=../target/surefire-reports/TEST-result.xml"
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-csharp.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+  e2e-nodejs-test:
+    if: ${{ success() }}
+    name: Nodejs E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: nodejs e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: nodejs
+              CMD: |
+                cd /root/code/nodejs/nacosnodejstest && npm install
+                mocha test --reporter mocha-junit-reporter --reporter-options mochaFile=../target/surefire-reports/TEST-report.xml
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-nodejs.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
+  e2e-python-test:
+    if: ${{ success() }}
+    name: Python E2E Test
+    needs: [docker, deploy]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ["cluster","standalone"]
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
+    steps:
+      - uses: apache/rocketmq-test-tool@java-dev
+        name: python e2e test
+        with:
+          yamlString: |
+            action: test
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}
+            API_VERSION: v1
+            KIND: Pod
+            RESTART_POLICY: Never
+            ENV:
+              WAIT_TIME: 900
+              REPO_NAME: ${{ env.TEST_REPO_NAME }}
+              CODE: https://github.com/${{ env.TEST_REPO_NAME }}
+              BRANCH: main
+              CODE_PATH: python
+              CMD: |
+                cd /root/code/python && pip3 install -r requirements.txt && source ~/.bashrc
+                cd nacospythontest && pytest --junitxml ../target/surefire-reports/TEST-report.xml test/*_test.py --log-cli-level=DEBUG
+              ALL_IP: null
+            CONTAINER:
+              IMAGE: cloudnativeofalibabacloud/test-runner:v0.0.4
+              RESOURCE_LIMITS:
+                cpu: 2
+                memory: 2Gi
+              RESOURCE_REQUIRE:
+                cpu: 2
+                memory: 2Gi
+      - uses: actions/upload-artifact@v3
+        if: always()
+        name: Upload test log
+        with:
+          name: testlog-${{ matrix.mode }}-python.txt
+          path: testlog.txt
+      - name: add markdown
+        if: always()
+        run: |
+          cat result.md >> $GITHUB_STEP_SUMMARY
+
 
   clean:
     if: always()
     name: Clean
-    needs: [list-version, e2e-test]
+    needs: [docker, e2e-java-test, e2e-go-test, e2e-cpp-test, e2e-csharp-test, e2e-nodejs-test, e2e-python-test]
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
+      fail-fast: false
       matrix:
         mode: ["cluster","standalone"]
-        version: ${{ fromJSON(needs.list-version.outputs.version-json) }}
+        version: ${{ fromJSON(needs.docker.outputs.version-json) }}
     steps:
-      - uses: apache/rocketmq-test-tool@136205caa6e97f81744e30adea114dd2f09cc55e
+      - uses: apache/rocketmq-test-tool@java-dev
         name: clean
         with:
-          action: "clean"
-          ask-config: "${{ secrets.ASK_CONFIG_VIRGINA }}"
-          test-version: "${{ matrix.version }}"
-          job-id: ${{ strategy.job-index }}
+          yamlString: |
+            action: clean
+            namespace: nacos-${{ github.run_id }}-${{ strategy.job-index }}
+            askConfig: ${{ secrets.ASK_CONFIG_VIRGINA }}