diff --git a/.github/workflows/build-tests.yml b/.github/workflows/build-tests.yml index 239f2b0f..1d40d1ab 100644 --- a/.github/workflows/build-tests.yml +++ b/.github/workflows/build-tests.yml @@ -7,7 +7,30 @@ on: - '.github/**' env: - UNITY_LICENSE: "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \nm0Db8UK+ktnOLJBtHybkfetpcKo=o/pUbSQAukz7+ZYAWhnA0AJbIlyyCPL7bKVEM2lVqbrXt7cyey+umkCXamuOgsWPVUKBMkXtMH8L\n5etLmD0getWIhTGhzOnDCk+gtIPfL4jMo9tkEuOCROQAXCci23VFscKcrkB+3X6h4wEOtA2APhOY\nB+wvC794o8/82ffjP79aVAi57rp3Wmzx+9pe9yMwoJuljAy2sc2tIMgdQGWVmOGBpQm3JqsidyzI\nJWG2kjnc7pDXK9pwYzXoKiqUqqrut90d+kQqRyv7MSZXR50HFqD/LI69h68b7P8Bjo3bPXOhNXGR\n9YCoemH6EkfCJxp2gIjzjWW+l2Hj2EsFQi8YXw==" + UNITY_LICENSE: + "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \nm0Db8UK+ktnOLJBtHybkfetpcKo=o/pUbSQAukz7+ZYAWhnA0AJbIlyyCPL7bKVEM2lVqbrXt7cyey+umkCXamuOgsWPVUKBMkXtMH8L\n5etLmD0getWIhTGhzOnDCk+gtIPfL4jMo9tkEuOCROQAXCci23VFscKcrkB+3X6h4wEOtA2APhOY\nB+wvC794o8/82ffjP79aVAi57rp3Wmzx+9pe9yMwoJuljAy2sc2tIMgdQGWVmOGBpQm3JqsidyzI\nJWG2kjnc7pDXK9pwYzXoKiqUqqrut90d+kQqRyv7MSZXR50HFqD/LI69h68b7P8Bjo3bPXOhNXGR\n9YCoemH6EkfCJxp2gIjzjWW+l2Hj2EsFQi8YXw==" jobs: buildForAllPlatformsUbuntu: @@ -16,6 +39,9 @@ jobs: strategy: fail-fast: false matrix: + cloudRunnerCluster: + # - local-docker + - local projectPath: - test-project unityVersion: @@ -62,6 +88,7 @@ jobs: unityVersion: ${{ matrix.unityVersion }} targetPlatform: ${{ matrix.targetPlatform }} customParameters: -profile SomeProfile -someBoolean -someValue exampleValue + cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }} ########################### # Upload # diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 10a5cfea..6568e057 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -29,7 +29,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: eu-west-2 - - run: yarn run cli -m aws-list-all + - run: yarn run cli -m list-resources env: AWS_REGION: eu-west-2 AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/cloud-runner-local-pipeline.yml b/.github/workflows/cloud-runner-local-pipeline.yml new file mode 100644 index 00000000..84c4ac08 --- /dev/null +++ b/.github/workflows/cloud-runner-local-pipeline.yml @@ -0,0 +1,96 @@ +name: Cloud Runner Local + +on: + push: { branches: ['!cloud-runner-develop', '!cloud-runner-preview', '!main'] } +# push: { branches: [main] } +# pull_request: +# paths-ignore: +# - '.github/**' + +jobs: + integrationTests: + name: Integration Tests + if: github.event.event_type != 'pull_request_target' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + cloudRunnerCluster: + - local-docker + targetPlatform: + - StandaloneWindows64 # Build a Windows 64-bit standalone. + # steps + steps: + - name: Checkout (default) + uses: actions/checkout@v2 + with: + lfs: true + - run: yarn + - run: yarn run cli --help + - run: yarn run test-i --detectOpenHandles --forceExit --runInBand + env: + UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} + PROJECT_PATH: ${{ matrix.projectPath }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TARGET_PLATFORM: ${{ matrix.targetPlatform }} + cloudRunnerTests: true + versioning: None + CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }} + buildTests: + name: Build Tests + if: github.event.event_type != 'pull_request_target' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + cloudRunnerCluster: + - local-docker + targetPlatform: + - StandaloneOSX # Build a macOS standalone (Intel 64-bit). + - StandaloneWindows64 # Build a Windows 64-bit standalone. + - StandaloneLinux64 # Build a Linux 64-bit standalone. + - WebGL # WebGL. + - iOS # Build an iOS player. + - Android # Build an Android .apk. + # - StandaloneWindows # Build a Windows standalone. + # - WSAPlayer # Build an Windows Store Apps player. + # - PS4 # Build a PS4 Standalone. + # - XboxOne # Build a Xbox One Standalone. + # - tvOS # Build to Apple's tvOS platform. + # - Switch # Build a Nintendo Switch player + # steps + steps: + - name: Checkout (default) + uses: actions/checkout@v2 + with: + lfs: true + - uses: ./ + id: unity-build + timeout-minutes: 25 + env: + CLOUD_RUNNER_BRANCH: ${{ github.ref }} + CLOUD_RUNNER_DEBUG: true + CLOUD_RUNNER_DEBUG_TREE: true + DEBUG: true + PROJECT_PATH: test-project + UNITY_VERSION: 2019.3.15f1 + USE_IL2CPP: false + with: + cloudRunnerTests: true + versioning: None + projectPath: ${{ matrix.projectPath }} + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + targetPlatform: ${{ matrix.targetPlatform }} + cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }} + - run: | + mv ./cloud-runner-cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 + ls + - run: yarn run cli -m list-resources + ########################### + # Upload # + ########################### + - uses: actions/upload-artifact@v2 + with: + name: AWS Build (${{ matrix.targetPlatform }}) + path: build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 + retention-days: 14 diff --git a/.github/workflows/cloud-runner-pipeline.yml b/.github/workflows/cloud-runner-pipeline.yml index 1b7c2293..70a8a8f0 100644 --- a/.github/workflows/cloud-runner-pipeline.yml +++ b/.github/workflows/cloud-runner-pipeline.yml @@ -1,11 +1,7 @@ -name: Cloud Runner +name: Cloud Runner CI Pipeline on: - push: { branches: [cloud-runner-develop, main] } -# push: { branches: [main] } -# pull_request: -# paths-ignore: -# - '.github/**' + push: { branches: [cloud-runner-develop, cloud-runner-preview] } env: GKE_ZONE: 'us-central1' @@ -21,43 +17,38 @@ env: AWS_DEFAULT_REGION: eu-west-2 AWS_BASE_STACK_NAME: game-ci-github-pipelines CLOUD_RUNNER_BRANCH: ${{ github.ref }} - CLOUD_RUNNER_TESTS: true + CLOUD_RUNNER_DEBUG: true + CLOUD_RUNNER_DEBUG_TREE: true DEBUG: true UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} + PROJECT_PATH: test-project + UNITY_VERSION: 2019.3.15f1 + USE_IL2CPP: false jobs: - awsBuild: - name: AWS Fargate Build - if: github.event.pull_request.draft == false + integrationTests: + name: Integration Tests + if: github.event.event_type != 'pull_request_target' runs-on: ubuntu-latest strategy: fail-fast: false matrix: - projectPath: - - test-project - unityVersion: - # - 2019.2.11f1 - - 2019.3.15f1 - targetPlatform: - #- StandaloneOSX # Build a macOS standalone (Intel 64-bit). - - StandaloneWindows64 # Build a Windows 64-bit standalone. - - StandaloneLinux64 # Build a Linux 64-bit standalone. - - WebGL # WebGL. - #- iOS # Build an iOS player. - #- Android # Build an Android .apk. - # - StandaloneWindows # Build a Windows standalone. - # - WSAPlayer # Build an Windows Store Apps player. - # - PS4 # Build a PS4 Standalone. - # - XboxOne # Build a Xbox One Standalone. - # - tvOS # Build to Apple's tvOS platform. - # - Switch # Build a Nintendo Switch player - # steps + cloudRunnerCluster: + - aws + - local-docker + - k8s steps: - name: Checkout (default) uses: actions/checkout@v2 - if: github.event.event_type != 'pull_request_target' with: lfs: true + - uses: google-github-actions/setup-gcloud@v0 + with: + version: '288.0.0' + service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }} + service_account_key: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} + - name: Get GKE cluster credentials + run: gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -66,81 +57,117 @@ jobs: aws-region: eu-west-2 - run: yarn - run: yarn run cli --help - - run: yarn run test "caching" - - run: yarn run test-i-aws + - run: yarn run test "cloud-runner-run-twice-retaining" --detectOpenHandles --forceExit --runInBand + if: matrix.CloudRunnerCluster == 'aws' || matrix.CloudRunnerCluster == 'k8s' + timeout-minutes: 180 env: UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} - PROJECT_PATH: ${{ matrix.projectPath }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TARGET_PLATFORM: ${{ matrix.targetPlatform }} + PROJECT_PATH: test-project + GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TARGET_PLATFORM: StandaloneWindows64 cloudRunnerTests: true versioning: None - - uses: ./ - id: aws-fargate-unity-build - timeout-minutes: 25 - with: - cloudRunnerCluster: aws + CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }} + - run: yarn run test-i --detectOpenHandles --forceExit --runInBand + if: matrix.CloudRunnerCluster == 'local-docker' + timeout-minutes: 180 + env: + UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} + PROJECT_PATH: test-project + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TARGET_PLATFORM: StandaloneWindows64 + cloudRunnerTests: true versioning: None - projectPath: ${{ matrix.projectPath }} - unityVersion: ${{ matrix.unityVersion }} - targetPlatform: ${{ matrix.targetPlatform }} - githubToken: ${{ secrets.GITHUB_TOKEN }} - postBuildSteps: | - - name: upload - image: amazon/aws-cli - commands: | - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default - aws configure set region $AWS_DEFAULT_REGION --profile default - aws s3 ls - aws s3 ls game-ci-test-storage - ls /data/cache/$CACHE_KEY - ls /data/cache/$CACHE_KEY/build - aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.tar - secrets: - - name: awsAccessKeyId - value: ${{ secrets.AWS_ACCESS_KEY_ID }} - - name: awsSecretAccessKey - value: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: awsDefaultRegion - value: eu-west-2 - - run: | - aws s3 cp s3://game-ci-test-storage/${{ steps.aws-fargate-unity-build.outputs.CACHE_KEY }}/build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.tar build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.tar - ls - - run: yarn run cli -m aws-garbage-collect - ########################### - # Upload # - ########################### - # download from cloud storage - - uses: actions/upload-artifact@v2 - with: - name: AWS Build (${{ matrix.targetPlatform }}) - path: build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.tar - retention-days: 14 - k8sBuilds: - name: K8s (GKE Autopilot) build for ${{ matrix.targetPlatform }} on version ${{ matrix.unityVersion }} + CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }} + + buildTargetTests: + name: Build Tests - Targets + if: github.event.event_type != 'pull_request_target' runs-on: ubuntu-latest strategy: fail-fast: false matrix: - unityVersion: - # - 2019.2.11f1 - - 2019.3.15f1 + cloudRunnerCluster: + #- aws + - local-docker + #- k8s targetPlatform: - # - StandaloneWindows64 - - StandaloneLinux64 + - StandaloneOSX # Build a macOS standalone (Intel 64-bit). + # - StandaloneWindows64 # Build a Windows 64-bit standalone. + - StandaloneLinux64 # Build a Linux 64-bit standalone. + - WebGL # WebGL. + - iOS # Build an iOS player. + - Android # Build an Android .apk. steps: - ########################### - # Checkout # - ########################### - - uses: actions/checkout@v2 - if: github.event.event_type != 'pull_request_target' + - name: Checkout (default) + uses: actions/checkout@v2 + with: + lfs: true + + - uses: google-github-actions/setup-gcloud@v0 + with: + version: '288.0.0' + service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }} + service_account_key: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }} + - name: Get GKE cluster credentials + run: gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 + - run: yarn + - uses: ./ + id: unity-build + timeout-minutes: 90 + env: + UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} + with: + cloudRunnerTests: true + versioning: None + projectPath: test-project + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + targetPlatform: ${{ matrix.targetPlatform }} + cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }} + customStepFiles: aws-s3-upload-build,aws-s3-pull-cache,aws-s3-upload-cache + - run: | + aws s3 cp s3://game-ci-test-storage/cloud-runner-cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 + ls + - run: yarn run cli -m list-resources + env: + cloudRunnerTests: true + CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }} + - uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }}) + path: build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 + retention-days: 14 + buildTests: + name: Build Tests - Providers + if: github.event.event_type != 'pull_request_target' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + cloudRunnerCluster: + - aws + - local-docker + - k8s + targetPlatform: + #- StandaloneOSX # Build a macOS standalone (Intel 64-bit). + - StandaloneWindows64 # Build a Windows 64-bit standalone. + #- StandaloneLinux64 # Build a Linux 64-bit standalone. + #- WebGL # WebGL. + #- iOS # Build an iOS player. + #- Android # Build an Android .apk. + # steps + steps: + - name: Checkout (default) + uses: actions/checkout@v2 with: lfs: true - ########################### - # Setup # - ########################### - uses: google-github-actions/setup-gcloud@v0 with: version: '288.0.0' @@ -149,70 +176,36 @@ jobs: - name: Get GKE cluster credentials run: gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT - ########################### - # Cloud Runner Test Suite # - ########################### - - uses: actions/setup-node@v2 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 with: - node-version: 12.x + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: eu-west-2 - run: yarn - - run: yarn run cli --help - - run: yarn run test "caching" - - name: Cloud Runner Test Suite - run: yarn run test-i-k8s --detectOpenHandles --forceExit + - uses: ./ + id: unity-build + timeout-minutes: 90 env: UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} - PROJECT_PATH: ${{ matrix.projectPath }} - TARGET_PLATFORM: ${{ matrix.targetPlatform }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - KUBE_CONFIG: ${{ steps.read-base64.outputs.base64 }} - unityVersion: ${{ matrix.unityVersion }} + with: cloudRunnerTests: true versioning: None - - ########################### - # Cloud Runner Build Test # - ########################### - - name: Cloud Runner Build Test - uses: ./ - id: k8s-unity-build - timeout-minutes: 30 - with: - cloudRunnerCluster: k8s - UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }} - targetPlatform: ${{ matrix.targetPlatform }} - kubeConfig: ${{ steps.read-base64.outputs.base64 }} - githubToken: ${{ secrets.GITHUB_TOKEN }} projectPath: test-project - unityVersion: ${{ matrix.unityVersion }} - versioning: None - postBuildSteps: | - - name: upload - image: amazon/aws-cli - commands: | - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default - aws configure set region $AWS_DEFAULT_REGION --profile default - aws s3 ls - aws s3 ls game-ci-test-storage - ls /data/cache/$CACHE_KEY - aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.tar - secrets: - - name: awsAccessKeyId - value: ${{ secrets.AWS_ACCESS_KEY_ID }} - - name: awsSecretAccessKey - value: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - name: awsDefaultRegion - value: eu-west-2 + gitPrivateToken: ${{ secrets.GITHUB_TOKEN }} + targetPlatform: ${{ matrix.targetPlatform }} + cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }} + customStepFiles: aws-s3-upload-build,aws-s3-pull-cache,aws-s3-upload-cache - run: | - aws s3 cp s3://game-ci-test-storage/${{ steps.k8s-unity-build.outputs.CACHE_KEY }}/build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.tar build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.tar + aws s3 cp s3://game-ci-test-storage/cloud-runner-cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 ls - ########################### - # Upload # - ########################### - # download from cloud storage + - run: yarn run cli -m list-resources + if: always() + env: + cloudRunnerTests: true + CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }} - uses: actions/upload-artifact@v2 with: - name: K8s Build (${{ matrix.targetPlatform }}) - path: build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.tar + name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }}) + path: build-${{ steps.unity-build.outputs.BUILD_GUID }}.tar.lz4 retention-days: 14 diff --git a/.vscode/settings.json b/.vscode/settings.json index e22e08b0..5a33b7a0 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,5 @@ { + "files.eol": "\n", "god.tsconfig": "./tsconfig.json", "yaml.customTags": [ "!And", diff --git a/action.yml b/action.yml index 6658a9a8..64effd18 100644 --- a/action.yml +++ b/action.yml @@ -9,7 +9,8 @@ inputs: unityVersion: required: false default: 'auto' - description: 'Version of unity to use for building the project. Use "auto" to get from your ProjectSettings/ProjectVersion.txt' + description: + 'Version of unity to use for building the project. Use "auto" to get from your ProjectSettings/ProjectVersion.txt' customImage: required: false default: '' @@ -81,91 +82,107 @@ inputs: gitPrivateToken: required: false default: '' - description: 'Github private token to pull from github' + description: '[CloudRunner] Github private token to pull from github' chownFilesTo: required: false default: '' - description: 'User and optionally group (user or user:group or uid:gid) to give ownership of the resulting build artifacts' + description: + 'User and optionally group (user or user:group or uid:gid) to give ownership of the resulting build artifacts' allowDirtyBuild: required: false default: '' - description: 'Allows the branch of the build to be dirty, and still generate the build.' + description: '[CloudRunner] Allows the branch of the build to be dirty, and still generate the build.' postBuildSteps: required: false default: '' - description: 'run a post build job in yaml format with the keys image, secrets (name, value object array), command string' + description: + '[CloudRunner] run a post build job in yaml format with the keys image, secrets (name, value object array), + command string' preBuildSteps: required: false default: '' - description: 'Run a pre build job after the repository setup but before the build job (in yaml format with the keys image, secrets (name, value object array), command line string)' + description: + '[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the + keys image, secrets (name, value object array), command line string)' + customStepFiles: + required: false + default: '' + description: + '[CloudRunner] Specify the names (by file name) of custom steps to run before or after cloud runner jobs, must + match a yaml step file inside your repo in the folder .game-ci/steps/' + customHookFiles: + required: false + default: '' + description: + '[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must + match a yaml step file inside your repo in the folder .game-ci/hooks/' customJobHooks: required: false default: '' - description: 'Specify custom commands and trigger hooks (injects commands into jobs)' + description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)' customJob: required: false default: '' - description: 'Run a custom job instead of the standard build automation for cloud runner (in yaml format with the keys image, secrets (name, value object array), command line string)' + description: + '[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the + keys image, secrets (name, value object array), command line string)' awsBaseStackName: default: 'game-ci' required: false - description: 'The Cloud Formation stack name that must be setup before using this option.' + description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.' cloudRunnerCluster: default: 'local' required: false - description: 'Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must be configured.' + description: + '[CloudRunner] Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must + be configured.' cloudRunnerCpu: default: '' required: false - description: 'Amount of CPU time to assign the remote build container' + description: '[CloudRunner] Amount of CPU time to assign the remote build container' cloudRunnerMemory: default: '' required: false - description: 'Amount of memory to assign the remote build container' - cachePushOverrideCommand: - default: '' - required: false - description: 'A command run every time a file is pushed to cache, formatted with input file path and remote cache path' - cachePullOverrideCommand: - default: '' - required: false - description: 'A command run every time before a file is being pulled from cache, formatted with request cache file and destination path' + description: '[CloudRunner] Amount of memory to assign the remote build container' readInputFromOverrideList: default: '' required: false - description: 'Comma separated list of input value names to read from "input override command"' + description: '[CloudRunner] Comma separated list of input value names to read from "input override command"' readInputOverrideCommand: default: '' required: false - description: 'Extend game ci by specifying a command to execute to pull input from external source e.g cloud provider secret managers' + description: + '[CloudRunner] Extend game ci by specifying a command to execute to pull input from external source e.g cloud + provider secret managers' kubeConfig: default: '' required: false - description: 'Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until completion.' + description: + '[CloudRunner] Supply a base64 encoded kubernetes config to run builds on kubernetes and stream logs until + completion.' kubeVolume: default: '' required: false - description: 'Supply a Persistent Volume Claim name to use for the Unity build.' + description: '[CloudRunner] Supply a Persistent Volume Claim name to use for the Unity build.' kubeStorageClass: default: '' required: false - description: 'Kubernetes storage class to use for cloud runner jobs, leave empty to install rook cluster.' + description: + '[CloudRunner] Kubernetes storage class to use for cloud runner jobs, leave empty to install rook cluster.' kubeVolumeSize: default: '5Gi' required: false - description: 'Amount of disc space to assign the Kubernetes Persistent Volume' + description: '[CloudRunner] Amount of disc space to assign the Kubernetes Persistent Volume' cacheKey: default: '' required: false - description: 'Cache key to indicate bucket for cache' - checkDependencyHealthOverride: - default: '' + description: '[CloudRunner] Cache key to indicate bucket for cache' + watchToEnd: + default: 'true' required: false - description: 'Use to specify a way to check depdency services health to enable resilient self-starting jobs' - startDependenciesOverride: - default: '' - required: false - description: 'Use to specify a way to start depdency services health to enable resilient self-starting jobs' + description: + '[CloudRunner] Whether or not to watch the build to the end. Can be used for especially long running jobs e.g + imports or self-hosted ephemeral runners.' outputs: volume: description: 'The Persistent Volume (PV) where the build artifacts have been stored by Kubernetes' diff --git a/dist/index.js b/dist/index.js index 1ccffaab..13bdcbc8 100644 Binary files a/dist/index.js and b/dist/index.js differ diff --git a/dist/index.js.map b/dist/index.js.map index a5fc6f39..ce98514a 100644 Binary files a/dist/index.js.map and b/dist/index.js.map differ diff --git a/game-ci/hooks/my-test-hook-post-build.yaml b/game-ci/hooks/my-test-hook-post-build.yaml new file mode 100644 index 00000000..b0c5d846 --- /dev/null +++ b/game-ci/hooks/my-test-hook-post-build.yaml @@ -0,0 +1,3 @@ +hook: after-build +commands: | + echo "after-build hook test!" diff --git a/game-ci/hooks/my-test-hook-pre-build.yaml b/game-ci/hooks/my-test-hook-pre-build.yaml new file mode 100644 index 00000000..b3294634 --- /dev/null +++ b/game-ci/hooks/my-test-hook-pre-build.yaml @@ -0,0 +1,3 @@ +hook: before-build +commands: | + echo "before-build hook test!!" diff --git a/game-ci/steps/my-test-step-post-build.yaml b/game-ci/steps/my-test-step-post-build.yaml new file mode 100644 index 00000000..496a1fb4 --- /dev/null +++ b/game-ci/steps/my-test-step-post-build.yaml @@ -0,0 +1,3 @@ +hook: after +commands: | + echo "after-build step test!" diff --git a/game-ci/steps/my-test-step-pre-build.yaml b/game-ci/steps/my-test-step-pre-build.yaml new file mode 100644 index 00000000..323a9265 --- /dev/null +++ b/game-ci/steps/my-test-step-pre-build.yaml @@ -0,0 +1,3 @@ +hook: before +commands: | + echo "before-build step test!" diff --git a/jest.config.js b/jest.config.js index be54f49e..d8024f48 100644 --- a/jest.config.js +++ b/jest.config.js @@ -18,6 +18,7 @@ module.exports = { transform: { '^.+\\.ts$': 'ts-jest', }, + autoRun: false, // Indicates whether each individual test should be reported during the run verbose: true, diff --git a/package.json b/package.json index dbb87ef5..ad2dc4f3 100644 --- a/package.json +++ b/package.json @@ -12,15 +12,15 @@ "lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts", "format": "prettier --write \"src/**/*.{js,ts}\"", "cli": "yarn ts-node src/index.ts -m cli", - "gcp-secrets-tests": "cross-env cloudRunnerCluster=aws cloudRunnerTests=true readInputOverrideCommand=\"gcloud secrets versions access 1 --secret=\"{0}\"\" populateOverride=true readInputFromOverrideList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"", - "gcp-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"gcloud secrets versions access 1 --secret=\"{0}\"\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD", - "aws-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"aws secretsmanager get-secret-value --secret-id {0}\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD", + "gcp-secrets-tests": "cross-env cloudRunnerCluster=aws cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" populateOverride=true readInputFromOverrideList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"", + "gcp-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD", + "aws-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD", "cli-aws": "cross-env cloudRunnerCluster=aws yarn run test-cli", "cli-k8s": "cross-env cloudRunnerCluster=k8s yarn run test-cli", "test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project", "test": "jest", - "test-i": "yarn run test-i-aws && yarn run test-i-k8s", - "test-i-f": "yarn run test-i-aws && yarn run test-i-k8s && yarn run cli-k8s && yarn run cli-aws", + "test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"", + "test-i-*": "yarn run test-i-aws && yarn run test-i-k8s", "test-i-aws": "cross-env cloudRunnerTests=true cloudRunnerCluster=aws yarn test -i -t \"cloud runner\"", "test-i-k8s": "cross-env cloudRunnerTests=true cloudRunnerCluster=k8s yarn test -i -t \"cloud runner\"" }, diff --git a/src/model/build-parameters.ts b/src/model/build-parameters.ts index 27c1aad9..c3c622a8 100644 --- a/src/model/build-parameters.ts +++ b/src/model/build-parameters.ts @@ -9,6 +9,8 @@ import Versioning from './versioning'; import { GitRepoReader } from './input-readers/git-repo'; import { GithubCliReader } from './input-readers/github-cli'; import { Cli } from './cli/cli'; +import GitHub from './github'; +import CloudRunnerOptions from './cloud-runner/cloud-runner-options'; class BuildParameters { public editorVersion!: string; @@ -45,12 +47,8 @@ class BuildParameters { public kubeStorageClass!: string; public chownFilesTo!: string; public customJobHooks!: string; - public cachePushOverrideCommand!: string; - public cachePullOverrideCommand!: string; public readInputFromOverrideList!: string; public readInputOverrideCommand!: string; - public checkDependencyHealthOverride!: string; - public startDependenciesOverride!: string; public cacheKey!: string; public postBuildSteps!: string; @@ -63,9 +61,15 @@ class BuildParameters { public logId!: string; public buildGuid!: string; public cloudRunnerBranch!: string; - public cloudRunnerIntegrationTests!: boolean; + public cloudRunnerDebug!: boolean; public cloudRunnerBuilderPlatform!: string | undefined; public isCliMode!: boolean; + public retainWorkspace!: boolean; + public maxRetainedWorkspaces!: number; + public useSharedLargePackages!: boolean; + public useLz4Compression!: boolean; + public garbageCollectionMaxAge!: number; + public constantGarbageCollection!: boolean; static async create(): Promise { const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidAppBundle); @@ -78,13 +82,13 @@ class BuildParameters { // --- let unitySerial = ''; if (Input.unityLicensingServer === '') { - if (!process.env.UNITY_SERIAL && Input.githubInputEnabled) { + if (!process.env.UNITY_SERIAL && GitHub.githubInputEnabled) { // No serial was present, so it is a personal license that we need to convert if (!process.env.UNITY_LICENSE) { throw new Error(`Missing Unity License File and no Serial was found. If this - is a personal license, make sure to follow the activation - steps and set the UNITY_LICENSE GitHub secret or enter a Unity - serial number inside the UNITY_SERIAL GitHub secret.`); + is a personal license, make sure to follow the activation + steps and set the UNITY_LICENSE GitHub secret or enter a Unity + serial number inside the UNITY_SERIAL GitHub secret.`); } unitySerial = this.getSerialFromLicenseFile(process.env.UNITY_LICENSE); } else { @@ -117,36 +121,38 @@ class BuildParameters { sshAgent: Input.sshAgent, gitPrivateToken: Input.gitPrivateToken || (await GithubCliReader.GetGitHubAuthToken()), chownFilesTo: Input.chownFilesTo, - cloudRunnerCluster: Input.cloudRunnerCluster, - cloudRunnerBuilderPlatform: Input.cloudRunnerBuilderPlatform, - awsBaseStackName: Input.awsBaseStackName, - kubeConfig: Input.kubeConfig, - cloudRunnerMemory: Input.cloudRunnerMemory, - cloudRunnerCpu: Input.cloudRunnerCpu, - kubeVolumeSize: Input.kubeVolumeSize, - kubeVolume: Input.kubeVolume, - postBuildSteps: Input.postBuildSteps, - preBuildSteps: Input.preBuildSteps, - customJob: Input.customJob, + cloudRunnerCluster: CloudRunnerOptions.cloudRunnerCluster, + cloudRunnerBuilderPlatform: CloudRunnerOptions.cloudRunnerBuilderPlatform, + awsBaseStackName: CloudRunnerOptions.awsBaseStackName, + kubeConfig: CloudRunnerOptions.kubeConfig, + cloudRunnerMemory: CloudRunnerOptions.cloudRunnerMemory, + cloudRunnerCpu: CloudRunnerOptions.cloudRunnerCpu, + kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize, + kubeVolume: CloudRunnerOptions.kubeVolume, + postBuildSteps: CloudRunnerOptions.postBuildSteps, + preBuildSteps: CloudRunnerOptions.preBuildSteps, + customJob: CloudRunnerOptions.customJob, runNumber: Input.runNumber, branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()), - cloudRunnerBranch: Input.cloudRunnerBranch.split('/').reverse()[0], - cloudRunnerIntegrationTests: Input.cloudRunnerTests, + cloudRunnerBranch: CloudRunnerOptions.cloudRunnerBranch.split('/').reverse()[0], + cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug, githubRepo: Input.githubRepo || (await GitRepoReader.GetRemote()) || 'game-ci/unity-builder', isCliMode: Cli.isCliMode, - awsStackName: Input.awsBaseStackName, + awsStackName: CloudRunnerOptions.awsBaseStackName, gitSha: Input.gitSha, logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(), buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform), - customJobHooks: Input.customJobHooks(), - cachePullOverrideCommand: Input.cachePullOverrideCommand(), - cachePushOverrideCommand: Input.cachePushOverrideCommand(), - readInputOverrideCommand: Input.readInputOverrideCommand(), - readInputFromOverrideList: Input.readInputFromOverrideList(), - kubeStorageClass: Input.kubeStorageClass, - checkDependencyHealthOverride: Input.checkDependencyHealthOverride, - startDependenciesOverride: Input.startDependenciesOverride, - cacheKey: Input.cacheKey, + customJobHooks: CloudRunnerOptions.customJobHooks(), + readInputOverrideCommand: CloudRunnerOptions.readInputOverrideCommand(), + readInputFromOverrideList: CloudRunnerOptions.readInputFromOverrideList(), + kubeStorageClass: CloudRunnerOptions.kubeStorageClass, + cacheKey: CloudRunnerOptions.cacheKey, + retainWorkspace: CloudRunnerOptions.retainWorkspaces, + useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages, + useLz4Compression: CloudRunnerOptions.useLz4Compression, + maxRetainedWorkspaces: CloudRunnerOptions.maxRetainedWorkspaces, + constantGarbageCollection: CloudRunnerOptions.constantGarbageCollection, + garbageCollectionMaxAge: CloudRunnerOptions.garbageCollectionMaxAge, }; } diff --git a/src/model/cli/cli.ts b/src/model/cli/cli.ts index de7854e7..392ed28f 100644 --- a/src/model/cli/cli.ts +++ b/src/model/cli/cli.ts @@ -5,10 +5,14 @@ import { ActionYamlReader } from '../input-readers/action-yaml'; import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger'; import CloudRunnerQueryOverride from '../cloud-runner/services/cloud-runner-query-override'; import { CliFunction, CliFunctionsRepository } from './cli-functions-repository'; -import { AwsCliCommands } from '../cloud-runner/providers/aws/commands/aws-cli-commands'; import { Caching } from '../cloud-runner/remote-client/caching'; import { LfsHashing } from '../cloud-runner/services/lfs-hashing'; import { RemoteClient } from '../cloud-runner/remote-client'; +import CloudRunnerOptionsReader from '../cloud-runner/services/cloud-runner-options-reader'; +import GitHub from '../github'; +import { TaskParameterSerializer } from '../cloud-runner/services/task-parameter-serializer'; +import { CloudRunnerFolders } from '../cloud-runner/services/cloud-runner-folders'; +import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system'; export class Cli { public static options; @@ -27,13 +31,13 @@ export class Cli { } public static InitCliMode() { - CliFunctionsRepository.PushCliFunctionSource(AwsCliCommands); + CliFunctionsRepository.PushCliFunctionSource(RemoteClient); CliFunctionsRepository.PushCliFunctionSource(Caching); CliFunctionsRepository.PushCliFunctionSource(LfsHashing); - CliFunctionsRepository.PushCliFunctionSource(RemoteClient); const program = new Command(); program.version('0.0.1'); - const properties = Object.getOwnPropertyNames(Input); + + const properties = CloudRunnerOptionsReader.GetProperties(); const actionYamlReader: ActionYamlReader = new ActionYamlReader(); for (const element of properties) { program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element)); @@ -48,6 +52,7 @@ export class Cli { program.option('--cachePushFrom ', 'cache push from source folder'); program.option('--cachePushTo ', 'cache push to caching folder'); program.option('--artifactName ', 'caching artifact name'); + program.option('--select