Cloud runner develop - latest fixes (#524)

Cloud runner develop - latest fixes (#524)

.github/workflows/build-tests-ubuntu.yml

@@ -49,7 +49,7 @@ jobs:
         exclude:
           - targetPlatform: Android
             unityVersion: 2022.2.7f1
-        cloudRunnerCluster:
+        providerStrategy:
           # - local-docker
           - local
         projectPath:
@@ -109,7 +109,7 @@ jobs:
           unityVersion: ${{ matrix.unityVersion }}
           targetPlatform: ${{ matrix.targetPlatform }}
           customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          providerStrategy: ${{ matrix.providerStrategy }}
 
       ###########################
       #          Upload         #

.github/workflows/cloud-runner-async-checks.yml

@@ -23,7 +23,7 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: eu-west-2
-  AWS_BASE_STACK_NAME: game-ci-github-pipelines
+  AWS_STACK_NAME: game-ci-github-pipelines
   CLOUD_RUNNER_BRANCH: ${{ github.ref }}
   CLOUD_RUNNER_DEBUG: true
   CLOUD_RUNNER_DEBUG_TREE: true
@@ -39,20 +39,21 @@ jobs:
     if: github.event.event_type != 'pull_request_target'
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout (default)
+      - timeout-minutes: 180
-        uses: actions/checkout@v3
-        with:
-          lfs: false
-      - run: yarn
-      - run: yarn run cli -m checks-update
-        timeout-minutes: 180
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
           PROJECT_PATH: test-project
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
           CLOUD_RUNNER_CLUSTER: local-docker
-          AWS_BASE_STACK_NAME: game-ci-github-pipelines
+          AWS_STACK_NAME: game-ci-github-pipelines
           CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
+        run: |
+          git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
+          cd unity-builder
+          yarn
+          ls
+          yarn run cli -m checks-update

.github/workflows/cloud-runner-ci-pipeline.yml

@@ -21,81 +21,140 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: eu-west-2
-  AWS_BASE_STACK_NAME: game-ci-team-pipelines
+  AWS_STACK_NAME: game-ci-team-pipelines
   CLOUD_RUNNER_BRANCH: ${{ github.ref }}
-  CLOUD_RUNNER_DEBUG: true
-  CLOUD_RUNNER_DEBUG_TREE: true
   DEBUG: true
   UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
   PROJECT_PATH: test-project
   UNITY_VERSION: 2019.3.15f1
   USE_IL2CPP: false
   USE_GKE_GCLOUD_AUTH_PLUGIN: true
+  GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
-  integrationTests:
+  smokeTests:
-    name: Integration Tests
+    name: Smoke Tests
     if: github.event.event_type != 'pull_request_target'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cloudRunnerCluster:
+        test:
-          - aws
+          #- 'cloud-runner-async-workflow'
+          - 'cloud-runner-caching'
+          # - 'cloud-runner-end2end-caching'
+          # - 'cloud-runner-end2end-retaining'
+          - 'cloud-runner-environment'
+          - 'cloud-runner-hooks'
+          - 'cloud-runner-local-persistence'
+          - 'cloud-runner-locking-core'
+          - 'cloud-runner-locking-get-locked'
+        providerStrategy:
+          #- aws
           - local-docker
-          - k8s
+          #- k8s
     steps:
       - name: Checkout (default)
         uses: actions/checkout@v3
         with:
           lfs: false
-      - uses: google-github-actions/auth@v1
-        with:
-          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
-      - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v1'
-      - name: Get GKE cluster credentials
-        run: |
-          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
-          gcloud components install gke-gcloud-auth-plugin
-          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-2
+      - uses: google-github-actions/auth@v1
+        if: matrix.providerStrategy == 'k8s'
+        with:
+          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: 'Set up Cloud SDK'
+        if: matrix.providerStrategy == 'k8s'
+        uses: 'google-github-actions/setup-gcloud@v1.1.0'
+      - name: Get GKE cluster credentials
+        if: matrix.providerStrategy == 'k8s'
+        run: |
+          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+          gcloud components install gke-gcloud-auth-plugin
+          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
       - run: yarn
-      - run: yarn run test "cloud-runner-async-workflow" --detectOpenHandles --forceExit --runInBand
+      - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
-        if: matrix.CloudRunnerCluster != 'local-docker'
+        timeout-minutes: 35
-        timeout-minutes: 180
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
           PROJECT_PATH: test-project
-          GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
+          CLOUD_RUNNER_CLUSTER: ${{ matrix.providerStrategy }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  tests:
-      - run: yarn run test-i --detectOpenHandles --forceExit --runInBand
+    # needs:
-        if: matrix.CloudRunnerCluster == 'local-docker'
+    #   - smokeTests
-        timeout-minutes: 180
+    #   - buildTargetTests
+    name: Integration Tests
+    if: github.event.event_type != 'pull_request_target'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        providerStrategy:
+          - aws
+          - local-docker
+          - k8s
+        test:
+          - 'cloud-runner-async-workflow'
+          #- 'cloud-runner-caching'
+          - 'cloud-runner-end2end-locking'
+          - 'cloud-runner-end2end-caching'
+          - 'cloud-runner-end2end-retaining'
+          - 'cloud-runner-environment'
+          #- 'cloud-runner-hooks'
+          - 'cloud-runner-s3-steps'
+          #- 'cloud-runner-local-persistence'
+          #- 'cloud-runner-locking-core'
+          #- 'cloud-runner-locking-get-locked'
+    steps:
+      - name: Checkout (default)
+        uses: actions/checkout@v2
+        with:
+          lfs: false
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+      - uses: google-github-actions/auth@v1
+        if: matrix.providerStrategy == 'k8s'
+        with:
+          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: 'Set up Cloud SDK'
+        if: matrix.providerStrategy == 'k8s'
+        uses: 'google-github-actions/setup-gcloud@v1.1.0'
+      - name: Get GKE cluster credentials
+        if: matrix.providerStrategy == 'k8s'
+        run: |
+          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+          gcloud components install gke-gcloud-auth-plugin
+          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
+      - run: yarn
+      - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
+        timeout-minutes: 60
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
           PROJECT_PATH: test-project
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
+          PROVIDER_STRATEGY: ${{ matrix.providerStrategy }}
-  localBuildTests:
+  buildTargetTests:
     name: Local Build Target Tests
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cloudRunnerCluster:
+        providerStrategy:
           #- aws
           - local-docker
           #- k8s
@@ -114,20 +173,18 @@ jobs:
       - run: yarn
       - uses: ./
         id: unity-build
-        timeout-minutes: 90
+        timeout-minutes: 30
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
         with:
           cloudRunnerTests: true
           versioning: None
-          projectPath: test-project
-          gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
           targetPlatform: ${{ matrix.targetPlatform }}
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          providerStrategy: ${{ matrix.providerStrategy }}
       - run: |
           cp ./cloud-runner-cache/cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/${{ steps.unity-build.outputs.BUILD_ARTIFACT }} ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
       - uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
+          name: ${{ matrix.providerStrategy }} Build (${{ matrix.targetPlatform }})
           path: ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
           retention-days: 14

action.yml

@@ -118,7 +118,7 @@ inputs:
     description:
       '[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the
       keys image, secrets (name, value object array), command line string)'
-  customStepFiles:
+  containerHookFiles:
     required: false
     default: ''
     description:
@@ -130,7 +130,7 @@ inputs:
     description:
       '[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must
       match a yaml step file inside your repo in the folder .game-ci/hooks/'
-  customJobHooks:
+  customCommandHooks:
     required: false
     default: ''
     description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)'
@@ -140,11 +140,11 @@ inputs:
     description:
       '[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the
       keys image, secrets (name, value object array), command line string)'
-  awsBaseStackName:
+  awsStackName:
     default: 'game-ci'
     required: false
     description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.'
-  cloudRunnerCluster:
+  providerStrategy:
     default: 'local'
     required: false
     description:

game-ci/command-hooks/my-test-hook-post-build.yaml

@@ -1,3 +1,3 @@
-hook: after-build
+hook: after
 commands: |
   echo "after-build hook test!"

game-ci/command-hooks/my-test-hook-pre-build.yaml

@@ -1,3 +1,3 @@
-hook: before-build
+hook: before
 commands: |
   echo "before-build hook test!!"

package.json

@@ -12,17 +12,17 @@
     "lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
     "format": "prettier --write \"src/**/*.{js,ts}\"",
     "cli": "yarn ts-node src/index.ts -m cli",
-    "gcp-secrets-tests": "cross-env cloudRunnerCluster=aws cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" populateOverride=true readInputFromOverrideList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
+    "gcp-secrets-tests": "cross-env providerStrategy=aws cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" populateOverride=true readInputFromOverrideList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
     "gcp-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
     "aws-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
-    "cli-aws": "cross-env cloudRunnerCluster=aws yarn run test-cli",
+    "cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
-    "cli-k8s": "cross-env cloudRunnerCluster=k8s yarn run test-cli",
+    "cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
     "test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
     "test": "jest",
     "test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"",
     "test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
-    "test-i-aws": "cross-env cloudRunnerTests=true cloudRunnerCluster=aws yarn test -i -t \"cloud runner\"",
+    "test-i-aws": "cross-env cloudRunnerTests=true providerStrategy=aws yarn test -i -t \"cloud runner\"",
-    "test-i-k8s": "cross-env cloudRunnerTests=true cloudRunnerCluster=k8s yarn test -i -t \"cloud runner\""
+    "test-i-k8s": "cross-env cloudRunnerTests=true providerStrategy=k8s yarn test -i -t \"cloud runner\""
   },
   "engines": {
     "node": ">=16.x"
@@ -44,7 +44,7 @@
     "reflect-metadata": "^0.1.13",
     "semver": "^7.3.5",
     "unity-changeset": "^2.0.0",
-    "uuid": "^8.3.2",
+    "uuid": "^9.0.0",
     "yaml": "^1.10.2"
   },
   "devDependencies": {

src/index.ts

@@ -19,7 +19,7 @@ async function runMain() {
     const buildParameters = await BuildParameters.create();
     const baseImage = new ImageTag(buildParameters);
 
-    if (buildParameters.cloudRunnerCluster === 'local') {
+    if (buildParameters.providerStrategy === 'local') {
       core.info('Building locally');
       await PlatformSetup.setup(buildParameters, actionFolder);
       if (process.platform === 'darwin') {

src/model/build-parameters.ts

@@ -1,7 +1,7 @@
 import { customAlphabet } from 'nanoid';
 import AndroidVersioning from './android-versioning';
-import CloudRunnerConstants from './cloud-runner/services/cloud-runner-constants';
+import CloudRunnerConstants from './cloud-runner/options/cloud-runner-constants';
-import CloudRunnerBuildGuid from './cloud-runner/services/cloud-runner-guid';
+import CloudRunnerBuildGuid from './cloud-runner/options/cloud-runner-guid';
 import Input from './input';
 import Platform from './platform';
 import UnityVersioning from './unity-versioning';
@@ -10,7 +10,8 @@ import { GitRepoReader } from './input-readers/git-repo';
 import { GithubCliReader } from './input-readers/github-cli';
 import { Cli } from './cli/cli';
 import GitHub from './github';
-import CloudRunnerOptions from './cloud-runner/cloud-runner-options';
+import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
+import CloudRunner from './cloud-runner/cloud-runner';
 
 class BuildParameters {
   // eslint-disable-next-line no-undef
@@ -41,24 +42,23 @@ class BuildParameters {
 
   public customParameters!: string;
   public sshAgent!: string;
-  public cloudRunnerCluster!: string;
+  public providerStrategy!: string;
-  public awsBaseStackName!: string;
   public gitPrivateToken!: string;
   public awsStackName!: string;
   public kubeConfig!: string;
-  public cloudRunnerMemory!: string | undefined;
+  public containerMemory!: string;
-  public cloudRunnerCpu!: string | undefined;
+  public containerCpu!: string;
   public kubeVolumeSize!: string;
   public kubeVolume!: string;
   public kubeStorageClass!: string;
   public chownFilesTo!: string;
-  public customJobHooks!: string;
+  public commandHooks!: string;
-  public readInputFromOverrideList!: string;
+  public pullInputList!: string[];
-  public readInputOverrideCommand!: string;
+  public inputPullCommand!: string;
   public cacheKey!: string;
 
-  public postBuildSteps!: string;
+  public postBuildContainerHooks!: string;
-  public preBuildSteps!: string;
+  public preBuildContainerHooks!: string;
   public customJob!: string;
   public runNumber!: string;
   public branch!: string;
@@ -68,17 +68,23 @@ class BuildParameters {
   public buildGuid!: string;
   public cloudRunnerBranch!: string;
   public cloudRunnerDebug!: boolean | undefined;
-  public cloudRunnerBuilderPlatform!: string | undefined;
+  public buildPlatform!: string | undefined;
   public isCliMode!: boolean;
-  public retainWorkspace!: boolean;
   public maxRetainedWorkspaces!: number;
-  public useSharedLargePackages!: boolean;
+  public useLargePackages!: boolean;
-  public useLz4Compression!: boolean;
+  public useCompressionStrategy!: boolean;
-  public garbageCollectionMaxAge!: number;
+  public garbageMaxAge!: number;
-  public constantGarbageCollection!: boolean;
   public githubChecks!: boolean;
+  public asyncWorkflow!: boolean;
+  public githubCheckId!: string;
+  public finalHooks!: string[];
+  public skipLfs!: boolean;
+  public skipCache!: boolean;
   public cacheUnityInstallationOnMac!: boolean;
   public unityHubVersionOnMac!: string;
+  public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
+    return buildParameters.maxRetainedWorkspaces > 0 && CloudRunner.lockedWorkspace !== ``;
+  }
 
   static async create(): Promise<BuildParameters> {
     const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidExportType);
@@ -144,16 +150,15 @@ class BuildParameters {
       sshAgent: Input.sshAgent,
       gitPrivateToken: Input.gitPrivateToken || (await GithubCliReader.GetGitHubAuthToken()),
       chownFilesTo: Input.chownFilesTo,
-      cloudRunnerCluster: CloudRunnerOptions.cloudRunnerCluster,
+      providerStrategy: CloudRunnerOptions.providerStrategy,
-      cloudRunnerBuilderPlatform: CloudRunnerOptions.cloudRunnerBuilderPlatform,
+      buildPlatform: CloudRunnerOptions.buildPlatform,
-      awsBaseStackName: CloudRunnerOptions.awsBaseStackName,
       kubeConfig: CloudRunnerOptions.kubeConfig,
-      cloudRunnerMemory: CloudRunnerOptions.cloudRunnerMemory,
+      containerMemory: CloudRunnerOptions.containerMemory,
-      cloudRunnerCpu: CloudRunnerOptions.cloudRunnerCpu,
+      containerCpu: CloudRunnerOptions.containerCpu,
       kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize,
       kubeVolume: CloudRunnerOptions.kubeVolume,
-      postBuildSteps: CloudRunnerOptions.postBuildSteps,
+      postBuildContainerHooks: CloudRunnerOptions.postBuildContainerHooks,
-      preBuildSteps: CloudRunnerOptions.preBuildSteps,
+      preBuildContainerHooks: CloudRunnerOptions.preBuildContainerHooks,
       customJob: CloudRunnerOptions.customJob,
       runNumber: Input.runNumber,
       branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
@@ -161,22 +166,25 @@ class BuildParameters {
       cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug,
       githubRepo: Input.githubRepo || (await GitRepoReader.GetRemote()) || 'game-ci/unity-builder',
       isCliMode: Cli.isCliMode,
-      awsStackName: CloudRunnerOptions.awsBaseStackName,
+      awsStackName: CloudRunnerOptions.awsStackName,
       gitSha: Input.gitSha,
       logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
       buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
-      customJobHooks: CloudRunnerOptions.customJobHooks(),
+      commandHooks: CloudRunnerOptions.commandHooks,
-      readInputOverrideCommand: CloudRunnerOptions.readInputOverrideCommand(),
+      inputPullCommand: CloudRunnerOptions.inputPullCommand,
-      readInputFromOverrideList: CloudRunnerOptions.readInputFromOverrideList(),
+      pullInputList: CloudRunnerOptions.pullInputList,
       kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
       cacheKey: CloudRunnerOptions.cacheKey,
-      retainWorkspace: CloudRunnerOptions.retainWorkspaces,
+      maxRetainedWorkspaces: Number.parseInt(CloudRunnerOptions.maxRetainedWorkspaces),
-      useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
+      useLargePackages: CloudRunnerOptions.useLargePackages,
-      useLz4Compression: CloudRunnerOptions.useLz4Compression,
+      useCompressionStrategy: CloudRunnerOptions.useCompressionStrategy,
-      maxRetainedWorkspaces: CloudRunnerOptions.maxRetainedWorkspaces,
+      garbageMaxAge: CloudRunnerOptions.garbageMaxAge,
-      constantGarbageCollection: CloudRunnerOptions.constantGarbageCollection,
-      garbageCollectionMaxAge: CloudRunnerOptions.garbageCollectionMaxAge,
       githubChecks: CloudRunnerOptions.githubChecks,
+      asyncWorkflow: CloudRunnerOptions.asyncCloudRunner,
+      githubCheckId: CloudRunnerOptions.githubCheckId,
+      finalHooks: CloudRunnerOptions.finalHooks,
+      skipLfs: CloudRunnerOptions.skipLfs,
+      skipCache: CloudRunnerOptions.skipCache,
       cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
       unityHubVersionOnMac: Input.unityHubVersionOnMac,
     };

src/model/cli/cli.ts

@@ -2,17 +2,16 @@ import { Command } from 'commander-ts';
 import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
 import * as core from '@actions/core';
 import { ActionYamlReader } from '../input-readers/action-yaml';
-import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
+import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
-import CloudRunnerQueryOverride from '../cloud-runner/services/cloud-runner-query-override';
+import CloudRunnerQueryOverride from '../cloud-runner/options/cloud-runner-query-override';
 import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
 import { Caching } from '../cloud-runner/remote-client/caching';
-import { LfsHashing } from '../cloud-runner/services/lfs-hashing';
+import { LfsHashing } from '../cloud-runner/services/utility/lfs-hashing';
 import { RemoteClient } from '../cloud-runner/remote-client';
-import CloudRunnerOptionsReader from '../cloud-runner/services/cloud-runner-options-reader';
+import CloudRunnerOptionsReader from '../cloud-runner/options/cloud-runner-options-reader';
 import GitHub from '../github';
-import { TaskParameterSerializer } from '../cloud-runner/services/task-parameter-serializer';
+import { CloudRunnerFolders } from '../cloud-runner/options/cloud-runner-folders';
-import { CloudRunnerFolders } from '../cloud-runner/services/cloud-runner-folders';
+import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
-import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
 import { OptionValues } from 'commander';
 import { InputKey } from '../input';
 
@@ -73,12 +72,14 @@ export class Cli {
     CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
     Cli.options!.versioning = 'None';
 
-    const buildParameter = TaskParameterSerializer.readBuildParameterFromEnvironment();
+    CloudRunner.buildParameters = await BuildParameters.create();
+    CloudRunner.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
     CloudRunnerLogger.log(`Build Params:
-      ${JSON.stringify(buildParameter, undefined, 4)}
+      ${JSON.stringify(CloudRunner.buildParameters, undefined, 4)}
     `);
-    CloudRunner.buildParameters = buildParameter;
+    CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
-    CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE;
+    CloudRunnerLogger.log(`Locked Workspace: ${CloudRunner.lockedWorkspace}`);
+    await CloudRunner.setup(CloudRunner.buildParameters);
 
     return await results.target[results.propertyKey](Cli.options);
   }
@@ -116,12 +117,16 @@ export class Cli {
   public static async asyncronousWorkflow(): Promise<string> {
     const buildParameter = await BuildParameters.create();
     const baseImage = new ImageTag(buildParameter);
+    await CloudRunner.setup(buildParameter);
 
     return await CloudRunner.run(buildParameter, baseImage.toString());
   }
 
   @CliFunction(`checks-update`, `runs a cloud runner build`)
   public static async checksUpdate() {
+    const buildParameter = await BuildParameters.create();
+
+    await CloudRunner.setup(buildParameter);
     const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
     core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
     if (input.mode === `create`) {
@@ -185,7 +190,7 @@ export class Cli {
       `build-${CloudRunner.buildParameters.buildGuid}`,
     );
 
-    if (!CloudRunner.buildParameters.retainWorkspace) {
+    if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
       await CloudRunnerSystem.Run(
         `rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
       );
@@ -193,21 +198,6 @@ export class Cli {
 
     await RemoteClient.runCustomHookFiles(`after-build`);
 
-    const parameters = await BuildParameters.create();
-    CloudRunner.setup(parameters);
-    if (parameters.constantGarbageCollection) {
-      await CloudRunnerSystem.Run(
-        `find /${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.buildVolumeFolder)}/ -name '*.*' -mmin +${
-          parameters.garbageCollectionMaxAge * 60
-        } -delete`,
-      );
-      await CloudRunnerSystem.Run(
-        `find ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForAllFull)} -name '*.*' -mmin +${
-          parameters.garbageCollectionMaxAge * 60
-        } -delete`,
-      );
-    }
-
     return new Promise((result) => result(``));
   }
 }

src/model/cloud-runner/cloud-runner.ts

@@ -1,34 +1,42 @@
 import AwsBuildPlatform from './providers/aws';
 import { BuildParameters, Input } from '..';
 import Kubernetes from './providers/k8s';
-import CloudRunnerLogger from './services/cloud-runner-logger';
+import CloudRunnerLogger from './services/core/cloud-runner-logger';
-import { CloudRunnerStepState } from './cloud-runner-step-state';
+import { CloudRunnerStepParameters } from './options/cloud-runner-step-parameters';
 import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
 import { CloudRunnerError } from './error/cloud-runner-error';
-import { TaskParameterSerializer } from './services/task-parameter-serializer';
+import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
 import * as core from '@actions/core';
-import CloudRunnerSecret from './services/cloud-runner-secret';
+import CloudRunnerSecret from './options/cloud-runner-secret';
 import { ProviderInterface } from './providers/provider-interface';
-import CloudRunnerEnvironmentVariable from './services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from './options/cloud-runner-environment-variable';
 import TestCloudRunner from './providers/test';
 import LocalCloudRunner from './providers/local';
 import LocalDockerCloudRunner from './providers/docker';
 import GitHub from '../github';
-import SharedWorkspaceLocking from './services/shared-workspace-locking';
+import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
+import { FollowLogStreamService } from './services/core/follow-log-stream-service';
 
 class CloudRunner {
   public static Provider: ProviderInterface;
   public static buildParameters: BuildParameters;
   private static defaultSecrets: CloudRunnerSecret[];
   private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
-  static lockedWorkspace: string | undefined;
+  static lockedWorkspace: string = ``;
   public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
-  public static githubCheckId: number | string;
+  public static get isCloudRunnerEnvironment() {
-
+    return process.env[`GITHUB_ACTIONS`] !== `true`;
-  public static setup(buildParameters: BuildParameters) {
+  }
+  public static get isCloudRunnerAsyncEnvironment() {
+    return process.env[`ASYNC_WORKFLOW`] === `true`;
+  }
+  public static async setup(buildParameters: BuildParameters) {
     CloudRunnerLogger.setup();
     CloudRunnerLogger.log(`Setting up cloud runner`);
     CloudRunner.buildParameters = buildParameters;
+    if (CloudRunner.buildParameters.githubCheckId === ``) {
+      CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
+    }
     CloudRunner.setupSelectedBuildPlatform();
     CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
     CloudRunner.cloudRunnerEnvironmentVariables =
@@ -46,15 +54,16 @@ class CloudRunner {
       core.setOutput(
         Input.ToEnvVarFormat(`buildArtifact`),
         `build-${CloudRunner.buildParameters.buildGuid}.tar${
-          CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+          CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
         }`,
       );
     }
+    FollowLogStreamService.Reset();
   }
 
   private static setupSelectedBuildPlatform() {
-    CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.cloudRunnerCluster}`);
+    CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.providerStrategy}`);
-    switch (CloudRunner.buildParameters.cloudRunnerCluster) {
+    switch (CloudRunner.buildParameters.providerStrategy) {
       case 'k8s':
         CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
         break;
@@ -74,14 +83,20 @@ class CloudRunner {
   }
 
   static async run(buildParameters: BuildParameters, baseImage: string) {
-    CloudRunner.setup(buildParameters);
+    await CloudRunner.setup(buildParameters);
+    if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
+    await CloudRunner.Provider.setupWorkflow(
+      CloudRunner.buildParameters.buildGuid,
+      CloudRunner.buildParameters,
+      CloudRunner.buildParameters.branch,
+      CloudRunner.defaultSecrets,
+    );
+    if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
     try {
-      CloudRunner.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
+      if (buildParameters.maxRetainedWorkspaces > 0) {
+        CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
 
-      if (buildParameters.retainWorkspace) {
+        const result = await SharedWorkspaceLocking.GetLockedWorkspace(
-        CloudRunner.lockedWorkspace = `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
-
-        const result = await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
           CloudRunner.lockedWorkspace,
           CloudRunner.buildParameters.buildGuid,
           CloudRunner.buildParameters,
@@ -95,21 +110,21 @@ class CloudRunner {
           ];
         } else {
           CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
-          buildParameters.retainWorkspace = false;
+          buildParameters.maxRetainedWorkspaces = 0;
-          CloudRunner.lockedWorkspace = undefined;
+          CloudRunner.lockedWorkspace = ``;
         }
       }
-      if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
+      const content = { ...CloudRunner.buildParameters };
-      await CloudRunner.Provider.setupWorkflow(
+      content.gitPrivateToken = ``;
-        CloudRunner.buildParameters.buildGuid,
+      content.unitySerial = ``;
-        CloudRunner.buildParameters,
+      const jsonContent = JSON.stringify(content, undefined, 4);
-        CloudRunner.buildParameters.branch,
+      await GitHub.updateGitHubCheck(jsonContent, CloudRunner.buildParameters.buildGuid);
-        CloudRunner.defaultSecrets,
-      );
-      if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
-      await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters.buildGuid);
       const output = await new WorkflowCompositionRoot().run(
-        new CloudRunnerStepState(baseImage, CloudRunner.cloudRunnerEnvironmentVariables, CloudRunner.defaultSecrets),
+        new CloudRunnerStepParameters(
+          baseImage,
+          CloudRunner.cloudRunnerEnvironmentVariables,
+          CloudRunner.defaultSecrets,
+        ),
       );
       if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Cleanup shared cloud runner resources');
       await CloudRunner.Provider.cleanupWorkflow(
@@ -122,22 +137,40 @@ class CloudRunner {
       if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
       await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
 
-      if (CloudRunner.buildParameters.retainWorkspace) {
+      if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
+        const workspace = CloudRunner.lockedWorkspace || ``;
         await SharedWorkspaceLocking.ReleaseWorkspace(
-          CloudRunner.lockedWorkspace || ``,
+          workspace,
           CloudRunner.buildParameters.buildGuid,
           CloudRunner.buildParameters,
         );
-        CloudRunner.lockedWorkspace = undefined;
+        const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, CloudRunner.buildParameters);
+        if (isLocked) {
+          throw new Error(
+            `still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
+              workspace,
+              buildParameters,
+            )}`,
+          );
+        }
+        CloudRunner.lockedWorkspace = ``;
       }
 
+      await GitHub.triggerWorkflowOnComplete(CloudRunner.buildParameters.finalHooks);
+
       if (buildParameters.constantGarbageCollection) {
-        CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageCollectionMaxAge, true, true);
+        CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
       }
 
       return output;
-    } catch (error) {
+    } catch (error: any) {
-      await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, error, `failure`, `completed`);
+      CloudRunnerLogger.log(JSON.stringify(error, undefined, 4));
+      await GitHub.updateGitHubCheck(
+        CloudRunner.buildParameters.buildGuid,
+        `Failed - Error ${error?.message || error}`,
+        `failure`,
+        `completed`,
+      );
       if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
       await CloudRunnerError.handleException(error, CloudRunner.buildParameters, CloudRunner.defaultSecrets);
       throw error;

src/model/cloud-runner/error/cloud-runner-error.ts

@@ -1,7 +1,7 @@
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
 import * as core from '@actions/core';
 import CloudRunner from '../cloud-runner';
-import CloudRunnerSecret from '../services/cloud-runner-secret';
+import CloudRunnerSecret from '../options/cloud-runner-secret';
 import BuildParameters from '../../build-parameters';
 
 export class CloudRunnerError {

src/model/cloud-runner/options/cloud-runner-folders.ts

@@ -1,6 +1,7 @@
 import path from 'node:path';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from './cloud-runner-options';
-import CloudRunner from './../cloud-runner';
+import CloudRunner from '../cloud-runner';
+import BuildParameters from '../../build-parameters';
 
 export class CloudRunnerFolders {
   public static readonly repositoryFolder = 'repo';
@@ -12,7 +13,7 @@ export class CloudRunnerFolders {
   // Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
 
   public static get uniqueCloudRunnerJobFolderAbsolute(): string {
-    return CloudRunner.buildParameters && CloudRunner.buildParameters.retainWorkspace && CloudRunner.lockedWorkspace
+    return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
       ? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
       : path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
   }

src/model/cloud-runner/options/cloud-runner-options-reader.ts

@@ -1,5 +1,5 @@
 import Input from '../../input';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from './cloud-runner-options';
 
 class CloudRunnerOptionsReader {
   static GetProperties() {

src/model/cloud-runner/options/cloud-runner-options.ts

@@ -1,6 +1,6 @@
-import { Cli } from '../cli/cli';
+import { Cli } from '../../cli/cli';
-import CloudRunnerQueryOverride from './services/cloud-runner-query-override';
+import CloudRunnerQueryOverride from './cloud-runner-query-override';
-import GitHub from '../github';
+import GitHub from '../../github';
 import * as core from '@actions/core';
 
 class CloudRunnerOptions {
@@ -58,7 +58,12 @@ class CloudRunnerOptions {
   // GitHub  parameters
   // ### ### ###
   static get githubChecks(): boolean {
-    return CloudRunnerOptions.getInput('githubChecks') === 'true' || false;
+    const value = CloudRunnerOptions.getInput('githubChecks');
+
+    return value === `true` || false;
+  }
+  static get githubCheckId(): string {
+    return CloudRunnerOptions.getInput('githubCheckId') || ``;
   }
 
   static get githubOwner(): string {
@@ -69,6 +74,10 @@ class CloudRunnerOptions {
     return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
   }
 
+  static get finalHooks(): string[] {
+    return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
+  }
+
   // ### ### ###
   // Git syncronization parameters
   // ### ### ###
@@ -76,59 +85,54 @@ class CloudRunnerOptions {
   static get githubRepo(): string | undefined {
     return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
   }
-
   static get branch(): string {
     if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
-      return CloudRunnerOptions.getInput(`GITHUB_REF`)!.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '');
+      return (
+        CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
+      );
     } else if (CloudRunnerOptions.getInput('branch')) {
-      return CloudRunnerOptions.getInput('branch')!;
+      return CloudRunnerOptions.getInput('branch') || ``;
     } else {
       return '';
     }
   }
 
-  static get gitSha(): string | undefined {
-    if (CloudRunnerOptions.getInput(`GITHUB_SHA`)) {
-      return CloudRunnerOptions.getInput(`GITHUB_SHA`)!;
-    } else if (CloudRunnerOptions.getInput(`GitSHA`)) {
-      return CloudRunnerOptions.getInput(`GitSHA`)!;
-    }
-  }
-
   // ### ### ###
   // Cloud Runner parameters
   // ### ### ###
 
-  static get cloudRunnerBuilderPlatform(): string | undefined {
+  static get buildPlatform(): string {
-    const input = CloudRunnerOptions.getInput('cloudRunnerBuilderPlatform');
+    const input = CloudRunnerOptions.getInput('buildPlatform');
     if (input) {
       return input;
     }
-    if (CloudRunnerOptions.cloudRunnerCluster !== 'local') {
+    if (CloudRunnerOptions.providerStrategy !== 'local') {
       return 'linux';
     }
 
-    return;
+    return ``;
   }
 
   static get cloudRunnerBranch(): string {
     return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
   }
 
-  static get cloudRunnerCluster(): string {
+  static get providerStrategy(): string {
+    const provider =
+      CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
     if (Cli.isCliMode) {
-      return CloudRunnerOptions.getInput('cloudRunnerCluster') || 'aws';
+      return provider || 'aws';
     }
 
-    return CloudRunnerOptions.getInput('cloudRunnerCluster') || 'local';
+    return provider || 'local';
   }
 
-  static get cloudRunnerCpu(): string | undefined {
+  static get containerCpu(): string {
-    return CloudRunnerOptions.getInput('cloudRunnerCpu');
+    return CloudRunnerOptions.getInput('containerCpu') || `1024`;
   }
 
-  static get cloudRunnerMemory(): string | undefined {
+  static get containerMemory(): string {
-    return CloudRunnerOptions.getInput('cloudRunnerMemory');
+    return CloudRunnerOptions.getInput('containerMemory') || `3072`;
   }
 
   static get customJob(): string {
@@ -139,40 +143,40 @@ class CloudRunnerOptions {
   // Custom commands from files parameters
   // ### ### ###
 
-  static get customStepFiles(): string[] {
+  static get containerHookFiles(): string[] {
-    return CloudRunnerOptions.getInput('customStepFiles')?.split(`,`) || [];
+    return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
   }
 
-  static get customHookFiles(): string[] {
+  static get commandHookFiles(): string[] {
-    return CloudRunnerOptions.getInput('customHookFiles')?.split(`,`) || [];
+    return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
   }
 
   // ### ### ###
   // Custom commands from yaml parameters
   // ### ### ###
 
-  static customJobHooks(): string {
+  static get commandHooks(): string {
-    return CloudRunnerOptions.getInput('customJobHooks') || '';
+    return CloudRunnerOptions.getInput('commandHooks') || '';
   }
 
-  static get postBuildSteps(): string {
+  static get postBuildContainerHooks(): string {
-    return CloudRunnerOptions.getInput('postBuildSteps') || '';
+    return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
   }
 
-  static get preBuildSteps(): string {
+  static get preBuildContainerHooks(): string {
-    return CloudRunnerOptions.getInput('preBuildSteps') || '';
+    return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
   }
 
   // ### ### ###
   // Input override handling
   // ### ### ###
 
-  static readInputFromOverrideList(): string {
+  static get pullInputList(): string[] {
-    return CloudRunnerOptions.getInput('readInputFromOverrideList') || '';
+    return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
   }
 
-  static readInputOverrideCommand(): string {
+  static get inputPullCommand(): string {
-    const value = CloudRunnerOptions.getInput('readInputOverrideCommand');
+    const value = CloudRunnerOptions.getInput('inputPullCommand');
 
     if (value === 'gcp-secret-manager') {
       return 'gcloud secrets versions access 1 --secret="{0}"';
@@ -187,8 +191,8 @@ class CloudRunnerOptions {
   // Aws
   // ### ### ###
 
-  static get awsBaseStackName(): string {
+  static get awsStackName() {
-    return CloudRunnerOptions.getInput('awsBaseStackName') || 'game-ci';
+    return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
   }
 
   // ### ### ###
@@ -204,7 +208,7 @@ class CloudRunnerOptions {
   }
 
   static get kubeVolumeSize(): string {
-    return CloudRunnerOptions.getInput('kubeVolumeSize') || '5Gi';
+    return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
   }
 
   static get kubeStorageClass(): string {
@@ -225,40 +229,34 @@ class CloudRunnerOptions {
 
   static get cloudRunnerDebug(): boolean {
     return (
-      CloudRunnerOptions.getInput(`cloudRunnerTests`) === 'true' ||
+      CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
-      CloudRunnerOptions.getInput(`cloudRunnerDebug`) === 'true' ||
+      CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
+      CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
+      CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
       false
     );
   }
-  static get cloudRunnerDebugTree(): string | boolean {
+  static get skipLfs(): boolean {
-    return CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) || false;
+    return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
   }
-  static get cloudRunnerDebugEnv(): string | boolean {
+  static get skipCache(): boolean {
-    return CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) || false;
+    return CloudRunnerOptions.getInput(`skipCache`) === `true`;
   }
 
-  static get watchCloudRunnerToEnd(): boolean {
+  public static get asyncCloudRunner(): boolean {
-    if (CloudRunnerOptions.asyncCloudRunner) {
+    return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
-      return false;
-    }
-
-    return CloudRunnerOptions.getInput(`watchToEnd`) === 'true' || true;
   }
 
-  static get asyncCloudRunner(): boolean {
+  public static get useLargePackages(): boolean {
-    return (CloudRunnerOptions.getInput('asyncCloudRunner') || `false`) === `true` || false;
+    return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
-  }
-
-  public static get useSharedLargePackages(): boolean {
-    return (CloudRunnerOptions.getInput(`useSharedLargePackages`) || 'false') === 'true';
   }
 
   public static get useSharedBuilder(): boolean {
-    return (CloudRunnerOptions.getInput(`useSharedBuilder`) || 'true') === 'true';
+    return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
   }
 
-  public static get useLz4Compression(): boolean {
+  public static get useCompressionStrategy(): boolean {
-    return (CloudRunnerOptions.getInput(`useLz4Compression`) || 'false') === 'true';
+    return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
   }
 
   public static get useCleanupCron(): boolean {
@@ -269,24 +267,16 @@ class CloudRunnerOptions {
   // Retained Workspace
   // ### ### ###
 
-  public static get retainWorkspaces(): boolean {
+  public static get maxRetainedWorkspaces(): string {
-    return CloudRunnerOptions.getInput(`retainWorkspaces`) === 'true' || false;
+    return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
-  }
-
-  static get maxRetainedWorkspaces(): number {
-    return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
   }
 
   // ### ### ###
   // Garbage Collection
   // ### ### ###
 
-  static get constantGarbageCollection(): boolean {
+  static get garbageMaxAge(): number {
-    return CloudRunnerOptions.getInput(`constantGarbageCollection`) === 'true' || true;
+    return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
-  }
-
-  static get garbageCollectionMaxAge(): number {
-    return Number(CloudRunnerOptions.getInput(`garbageCollectionMaxAge`)) || 24;
   }
 }
 

src/model/cloud-runner/options/cloud-runner-query-override.ts

@@ -1,6 +1,6 @@
 import Input from '../../input';
 import { GenericInputReader } from '../../input-readers/generic-input-reader';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from './cloud-runner-options';
 
 const formatFunction = (value: string, arguments_: any[]) => {
   for (const element of arguments_) {
@@ -31,11 +31,11 @@ class CloudRunnerQueryOverride {
   }
 
   private static shouldUseOverride(query: string) {
-    if (CloudRunnerOptions.readInputOverrideCommand() !== '') {
+    if (CloudRunnerOptions.inputPullCommand !== '') {
-      if (CloudRunnerOptions.readInputFromOverrideList() !== '') {
+      if (CloudRunnerOptions.pullInputList.length > 0) {
         const doesInclude =
-          CloudRunnerOptions.readInputFromOverrideList().split(',').includes(query) ||
+          CloudRunnerOptions.pullInputList.includes(query) ||
-          CloudRunnerOptions.readInputFromOverrideList().split(',').includes(Input.ToEnvVarFormat(query));
+          CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
 
         return doesInclude ? true : false;
       } else {
@@ -50,12 +50,12 @@ class CloudRunnerQueryOverride {
     }
 
     return await GenericInputReader.Run(
-      formatFunction(CloudRunnerOptions.readInputOverrideCommand(), [{ key: 0, value: query }]),
+      formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
     );
   }
 
   public static async PopulateQueryOverrideInput() {
-    const queries = CloudRunnerOptions.readInputFromOverrideList().split(',');
+    const queries = CloudRunnerOptions.pullInputList;
     CloudRunnerQueryOverride.queryOverrides = {};
     for (const element of queries) {
       if (CloudRunnerQueryOverride.shouldUseOverride(element)) {

src/model/cloud-runner/options/cloud-runner-step-parameters.ts

@@ -1,7 +1,7 @@
-import CloudRunnerEnvironmentVariable from './services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
-import CloudRunnerSecret from './services/cloud-runner-secret';
+import CloudRunnerSecret from './cloud-runner-secret';
 
-export class CloudRunnerStepState {
+export class CloudRunnerStepParameters {
   public image: string;
   public environment: CloudRunnerEnvironmentVariable[];
   public secrets: CloudRunnerSecret[];

src/model/cloud-runner/providers/aws/aws-base-stack.ts

@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import * as core from '@actions/core';
 import * as SDK from 'aws-sdk';
 import { BaseStackFormation } from './cloud-formations/base-stack-formation';

src/model/cloud-runner/providers/aws/aws-error.ts

@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import * as SDK from 'aws-sdk';
 import * as core from '@actions/core';
 import CloudRunner from '../../cloud-runner';

src/model/cloud-runner/providers/aws/aws-job-stack.ts

@@ -1,12 +1,12 @@
 import * as SDK from 'aws-sdk';
 import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { AWSError } from './aws-error';
 import CloudRunner from '../../cloud-runner';
 import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
-import CloudRunnerOptions from '../../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
 import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
 
 export class AWSJobStack {
@@ -27,21 +27,19 @@ export class AWSJobStack {
   ): Promise<CloudRunnerAWSTaskDef> {
     const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
     let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
-    const cpu = CloudRunner.buildParameters.cloudRunnerCpu || '1024';
-    const memory = CloudRunner.buildParameters.cloudRunnerMemory || '3072';
     taskDefCloudFormation = taskDefCloudFormation.replace(
       `ContainerCpu:
     Default: 1024`,
       `ContainerCpu:
-    Default: ${Number.parseInt(cpu)}`,
+    Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
     );
     taskDefCloudFormation = taskDefCloudFormation.replace(
       `ContainerMemory:
     Default: 2048`,
       `ContainerMemory:
-    Default: ${Number.parseInt(memory)}`,
+    Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
     );
-    if (CloudRunnerOptions.watchCloudRunnerToEnd) {
+    if (!CloudRunnerOptions.asyncCloudRunner) {
       taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
         taskDefCloudFormation,
         '# template resources logstream',
@@ -116,7 +114,7 @@ export class AWSJobStack {
       ...secretsMappedToCloudFormationParameters,
     ];
     CloudRunnerLogger.log(
-      `Starting AWS job with memory: ${CloudRunner.buildParameters.cloudRunnerMemory} cpu: ${CloudRunner.buildParameters.cloudRunnerCpu}`,
+      `Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
     );
     let previousStackExists = true;
     while (previousStackExists) {
@@ -140,11 +138,16 @@ export class AWSJobStack {
       Capabilities: ['CAPABILITY_IAM'],
       Parameters: parameters,
     };
-
     try {
       CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
       await CF.createStack(createStackInput).promise();
       await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
+      const describeStack = await CF.describeStacks({ StackName: taskDefStackName }).promise();
+      for (const parameter of parameters) {
+        if (!describeStack.Stacks?.[0].Parameters?.some((x) => x.ParameterKey === parameter.ParameterKey)) {
+          throw new Error(`Parameter ${parameter.ParameterKey} not found in stack`);
+        }
+      }
     } catch (error) {
       await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
       throw error;
@@ -180,7 +183,7 @@ export class AWSJobStack {
     if (CloudRunnerOptions.useCleanupCron) {
       try {
         CloudRunnerLogger.log(`Creating job cleanup formation`);
-        CF.createStack(createCleanupStackInput).promise();
+        await CF.createStack(createCleanupStackInput).promise();
 
         // await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
       } catch (error) {

src/model/cloud-runner/providers/aws/aws-task-runner.ts

@@ -1,14 +1,14 @@
 import * as AWS from 'aws-sdk';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
 import * as core from '@actions/core';
 import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
 import * as zlib from 'node:zlib';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { Input } from '../../..';
 import CloudRunner from '../../cloud-runner';
-import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
+import { CommandHookService } from '../../services/hooks/command-hook-service';
-import { FollowLogStreamService } from '../../services/follow-log-stream-service';
+import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
-import CloudRunnerOptions from '../../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
 import GitHub from '../../../github';
 
 class AWSTaskRunner {
@@ -32,7 +32,7 @@ class AWSTaskRunner {
     const streamName =
       taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
 
-    const task = await AWSTaskRunner.ECS.runTask({
+    const runParameters = {
       cluster,
       taskDefinition,
       platformVersion: '1.4.0',
@@ -41,7 +41,7 @@ class AWSTaskRunner {
           {
             name: taskDef.taskDefStackName,
             environment,
-            command: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
+            command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
           },
         ],
       },
@@ -53,16 +53,23 @@ class AWSTaskRunner {
           securityGroups: [ContainerSecurityGroup],
         },
       },
-    }).promise();
+    };
+
+    if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
+      CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
+      throw new Error(`Container Overrides length must be at most 8192`);
+    }
+
+    const task = await AWSTaskRunner.ECS.runTask(runParameters).promise();
     const taskArn = task.tasks?.[0].taskArn || '';
     CloudRunnerLogger.log('Cloud runner job is starting');
     await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
     CloudRunnerLogger.log(
-      `Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Watch:${
+      `Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
-        CloudRunnerOptions.watchCloudRunnerToEnd
+        CloudRunnerOptions.asyncCloudRunner
-      } Async:${CloudRunnerOptions.asyncCloudRunner}`,
+      }`,
     );
-    if (!CloudRunnerOptions.watchCloudRunnerToEnd) {
+    if (CloudRunnerOptions.asyncCloudRunner) {
       const shouldCleanup: boolean = false;
       const output: string = '';
       CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
@@ -72,26 +79,31 @@ class AWSTaskRunner {
 
     CloudRunnerLogger.log(`Streaming...`);
     const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
-    await new Promise((resolve) => resolve(5000));
+    let exitCode;
-    const taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
+    let containerState;
-    const containerState = taskData.containers?.[0];
+    let taskData;
-    const exitCode = containerState?.exitCode || undefined;
+    while (exitCode === undefined) {
+      await new Promise((resolve) => resolve(10000));
+      taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
+      containerState = taskData.containers?.[0];
+      exitCode = containerState?.exitCode;
+    }
     CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
-    const wasSuccessful = exitCode === 0 || (exitCode === undefined && taskData.lastStatus === 'RUNNING');
+    if (exitCode === undefined) {
+      CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
+    }
+    const wasSuccessful = exitCode === 0;
     if (wasSuccessful) {
       CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
 
       return { output, shouldCleanup };
-    } else {
-      if (taskData.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
-        throw new Error('Container exited with code 1');
-      }
-      const message = `Cloud runner job exit code ${exitCode}`;
-      taskData.overrides = undefined;
-      taskData.attachments = undefined;
-      CloudRunnerLogger.log(`${message} ${JSON.stringify(taskData, undefined, 4)}`);
-      throw new Error(message);
     }
+
+    if (taskData?.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
+      throw new Error('Container exited with code 1');
+    }
+
+    throw new Error(`Task failed`);
   }
 
   private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
@@ -129,7 +141,7 @@ class AWSTaskRunner {
     const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
     let iterator = await AWSTaskRunner.getLogIterator(stream);
 
-    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
+    const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
     CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
     await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch:  ${logBaseUrl}`, ``);
     let shouldReadLogs = true;

src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts

@@ -30,7 +30,7 @@ Parameters:
     Type: Number
     Description: How much CPU to give the container. 1024 is 1 CPU
   ContainerMemory:
-    Default: 2048
+    Default: 4096
     Type: Number
     Description: How much memory in megabytes to give the container
   BUILDGUID:

src/model/cloud-runner/providers/aws/index.ts

@@ -1,11 +1,11 @@
 import * as SDK from 'aws-sdk';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
 import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
 import AwsTaskRunner from './aws-task-runner';
 import { ProviderInterface } from '../provider-interface';
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { AWSJobStack as AwsJobStack } from './aws-job-stack';
 import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
 import { Input } from '../../..';
@@ -13,13 +13,13 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 import { TaskService } from './services/task-service';
-import CloudRunnerOptions from '../../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
 
 class AWSBuildEnvironment implements ProviderInterface {
   private baseStackName: string;
 
   constructor(buildParameters: BuildParameters) {
-    this.baseStackName = buildParameters.awsBaseStackName;
+    this.baseStackName = buildParameters.awsStackName;
   }
   async listResources(): Promise<ProviderResource[]> {
     await TaskService.getCloudFormationJobStacks();
@@ -75,7 +75,11 @@ class AWSBuildEnvironment implements ProviderInterface {
     branchName: string,
     // eslint-disable-next-line no-unused-vars
     defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
-  ) {}
+  ) {
+    process.env.AWS_REGION = Input.region;
+    const CF = new SDK.CloudFormation();
+    await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
+  }
 
   async runTaskInWorkflow(
     buildGuid: string,
@@ -94,8 +98,6 @@ class AWSBuildEnvironment implements ProviderInterface {
     CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
     const entrypoint = ['/bin/sh'];
     const startTimeMs = Date.now();
-
-    await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
     const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
       CF,
       buildGuid,
@@ -143,6 +145,9 @@ class AWSBuildEnvironment implements ProviderInterface {
     await CF.waitFor('stackDeleteComplete', {
       StackName: taskDef.taskDefStackName,
     }).promise();
+    await CF.waitFor('stackDeleteComplete', {
+      StackName: `${taskDef.taskDefStackName}-cleanup`,
+    }).promise();
     CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
     CloudRunnerLogger.log('Cleanup complete');
   }

src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts

@@ -1,6 +1,6 @@
 import AWS from 'aws-sdk';
 import Input from '../../../../input';
-import CloudRunnerLogger from '../../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
 import { TaskService } from './task-service';
 
 export class GarbageCollectionService {

src/model/cloud-runner/providers/aws/services/task-service.ts

@@ -1,6 +1,6 @@
 import AWS from 'aws-sdk';
 import Input from '../../../../input';
-import CloudRunnerLogger from '../../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
 import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
 import AwsTaskRunner from '../aws-task-runner';
 import { ListObjectsRequest } from 'aws-sdk/clients/s3';
@@ -161,7 +161,7 @@ export class TaskService {
     process.env.AWS_REGION = Input.region;
     const s3 = new AWS.S3();
     const listRequest: ListObjectsRequest = {
-      Bucket: CloudRunner.buildParameters.awsBaseStackName,
+      Bucket: CloudRunner.buildParameters.awsStackName,
     };
     const results = await s3.listObjects(listRequest).promise();
 

src/model/cloud-runner/providers/docker/index.ts

@@ -1,20 +1,21 @@
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { ProviderInterface } from '../provider-interface';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import Docker from '../../../docker';
 import { Action } from '../../..';
-import { writeFileSync } from 'fs';
+import { writeFileSync } from 'node:fs';
 import CloudRunner from '../../cloud-runner';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
-import { CloudRunnerSystem } from '../../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
-import fs from 'node:fs';
+import * as fs from 'node:fs';
+import { CommandHookService } from '../../services/hooks/command-hook-service';
 import { StringKeyValuePair } from '../../../shared-types';
 
 class LocalDockerCloudRunner implements ProviderInterface {
-  public buildParameters: BuildParameters | undefined;
+  public buildParameters!: BuildParameters;
 
   listResources(): Promise<ProviderResource[]> {
     return new Promise((resolve) => resolve([]));
@@ -51,14 +52,14 @@ class LocalDockerCloudRunner implements ProviderInterface {
     if (
       fs.existsSync(
         `${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
-          CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+          CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
         }`,
       )
     ) {
       await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
       await CloudRunnerSystem.Run(
         `rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
-          CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+          CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
         }`,
       );
     }
@@ -118,7 +119,7 @@ set -e
 mkdir -p /github/workspace/cloud-runner-cache
 mkdir -p /data/cache
 cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
-${commands}
+${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
 cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
 `;
     writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
@@ -149,6 +150,7 @@ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
         },
       },
       true,
+      false,
     );
 
     return myOutput;

src/model/cloud-runner/providers/k8s/index.ts

@@ -2,19 +2,18 @@ import * as k8s from '@kubernetes/client-node';
 import { BuildParameters } from '../../..';
 import * as core from '@actions/core';
 import { ProviderInterface } from '../provider-interface';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import KubernetesStorage from './kubernetes-storage';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
 import KubernetesTaskRunner from './kubernetes-task-runner';
 import KubernetesSecret from './kubernetes-secret';
 import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
 import KubernetesServiceAccount from './kubernetes-service-account';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { CoreV1Api } from '@kubernetes/client-node';
 import CloudRunner from '../../cloud-runner';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
-import KubernetesPods from './kubernetes-pods';
 
 class Kubernetes implements ProviderInterface {
   public static Instance: Kubernetes;
@@ -94,16 +93,8 @@ class Kubernetes implements ProviderInterface {
   ) {
     try {
       this.buildParameters = buildParameters;
-      const id = buildParameters.retainWorkspace ? CloudRunner.lockedWorkspace : buildParameters.buildGuid;
+      this.cleanupCronJobName = `unity-builder-cronjob-${buildParameters.buildGuid}`;
-      this.pvcName = `unity-builder-pvc-${id}`;
-      this.cleanupCronJobName = `unity-builder-cronjob-${id}`;
       this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
-      await KubernetesStorage.createPersistentVolumeClaim(
-        buildParameters,
-        this.pvcName,
-        this.kubeClient,
-        this.namespace,
-      );
 
       await KubernetesServiceAccount.createServiceAccount(this.serviceAccountName, this.namespace, this.kubeClient);
     } catch (error) {
@@ -124,74 +115,99 @@ class Kubernetes implements ProviderInterface {
       CloudRunnerLogger.log('Cloud Runner K8s workflow!');
 
       // Setup
+      const id = BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
+        ? CloudRunner.lockedWorkspace
+        : this.buildParameters.buildGuid;
+      this.pvcName = `unity-builder-pvc-${id}`;
+      await KubernetesStorage.createPersistentVolumeClaim(
+        this.buildParameters,
+        this.pvcName,
+        this.kubeClient,
+        this.namespace,
+      );
       this.buildGuid = buildGuid;
       this.secretName = `build-credentials-${this.buildGuid}`;
       this.jobName = `unity-builder-job-${this.buildGuid}`;
       this.containerName = `main`;
       await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
-      await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
-      this.setPodNameAndContainerName(await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
-      CloudRunnerLogger.log('Watching pod until running');
-      await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
       let output = '';
-      // eslint-disable-next-line no-constant-condition
+      try {
-      while (true) {
+        CloudRunnerLogger.log('Job does not exist');
-        try {
+        await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
-          CloudRunnerLogger.log('Pod running, streaming logs');
+        CloudRunnerLogger.log('Watching pod until running');
-          output = await KubernetesTaskRunner.runTask(
+        await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
-            this.kubeConfig,
-            this.kubeClient,
-            this.jobName,
-            this.podName,
-            'main',
-            this.namespace,
-          );
-          const running = await KubernetesPods.IsPodRunning(this.podName, this.namespace, this.kubeClient);
 
-          if (!running) {
+        CloudRunnerLogger.log('Pod running, streaming logs');
-            CloudRunnerLogger.log(`Pod not found, assumed ended!`);
+        CloudRunnerLogger.log(
-            break;
+          `Starting logs follow for pod: ${this.podName} container: ${this.containerName} namespace: ${this.namespace} pvc: ${this.pvcName} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
-          } else {
+        );
-            CloudRunnerLogger.log('Pod still running, recovering stream...');
+        output += await KubernetesTaskRunner.runTask(
-          }
+          this.kubeConfig,
-          await this.cleanupTaskResources();
+          this.kubeClient,
-        } catch (error: any) {
+          this.jobName,
-          let errorParsed;
+          this.podName,
-          try {
+          this.containerName,
-            errorParsed = JSON.parse(error);
+          this.namespace,
-          } catch {
+        );
-            errorParsed = error;
+      } catch (error: any) {
-          }
+        CloudRunnerLogger.log(`error running k8s workflow ${error}`);
-
+        await new Promise((resolve) => setTimeout(resolve, 3000));
-          const reason = errorParsed.reason || errorParsed.response?.body?.reason || ``;
+        CloudRunnerLogger.log(
-          const errorMessage = errorParsed.message || reason;
+          JSON.stringify(
-
+            (await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
-          const continueStreaming =
+              .map((x) => {
-            errorMessage.includes(`dial timeout, backstop`) ||
+                return {
-            errorMessage.includes(`HttpError: HTTP request failed`) ||
+                  message: x.message || ``,
-            errorMessage.includes(`an error occurred when try to find container`) ||
+                  name: x.metadata.name || ``,
-            errorMessage.includes(`not found`) ||
+                  reason: x.reason || ``,
-            errorMessage.includes(`Not Found`);
+                };
-          if (continueStreaming) {
+              })
-            CloudRunnerLogger.log('Log Stream Container Not Found');
+              .filter((x) => x.name.includes(this.podName)),
-            await new Promise((resolve) => resolve(5000));
+            undefined,
-            continue;
+            4,
-          } else {
+          ),
-            CloudRunnerLogger.log(`error running k8s workflow ${error}`);
+        );
-            throw error;
+        await this.cleanupTaskResources();
-          }
+        throw error;
-        }
       }
 
+      await this.cleanupTaskResources();
+
       return output;
     } catch (error) {
       CloudRunnerLogger.log('Running job failed');
       core.error(JSON.stringify(error, undefined, 4));
-      await this.cleanupTaskResources();
+
+      // await this.cleanupTaskResources();
       throw error;
     }
   }
 
+  private async createJob(
+    commands: string,
+    image: string,
+    mountdir: string,
+    workingdir: string,
+    environment: CloudRunnerEnvironmentVariable[],
+    secrets: CloudRunnerSecret[],
+  ) {
+    await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
+    const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
+    this.setPodNameAndContainerName(find);
+  }
+
+  private async doesJobExist(name: string) {
+    const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
+
+    return jobs.body.items.some((x) => x.metadata?.name === name);
+  }
+
+  private async doesFailedJobExist() {
+    const podStatus = await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
+
+    return podStatus.body.status?.phase === `Failed`;
+  }
+
   private async createNamespacedJob(
     commands: string,
     image: string,
@@ -215,14 +231,15 @@ class Kubernetes implements ProviderInterface {
           this.pvcName,
           this.jobName,
           k8s,
+          this.containerName,
         );
         await new Promise((promise) => setTimeout(promise, 15000));
-        await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
+        const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
         CloudRunnerLogger.log(`Build job created`);
         await new Promise((promise) => setTimeout(promise, 5000));
         CloudRunnerLogger.log('Job created');
 
-        return;
+        return result.body.metadata?.name;
       } catch (error) {
         CloudRunnerLogger.log(`Error occured creating job: ${error}`);
         throw error;
@@ -232,7 +249,7 @@ class Kubernetes implements ProviderInterface {
 
   setPodNameAndContainerName(pod: k8s.V1Pod) {
     this.podName = pod.metadata?.name || '';
-    this.containerName = pod.status?.containerStatuses?.[0].name || '';
+    this.containerName = pod.status?.containerStatuses?.[0].name || this.containerName;
   }
 
   async cleanupTaskResources() {
@@ -265,7 +282,7 @@ class Kubernetes implements ProviderInterface {
     // eslint-disable-next-line no-unused-vars
     defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
   ) {
-    if (buildParameters.retainWorkspace) {
+    if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
       return;
     }
     CloudRunnerLogger.log(`deleting PVC`);

src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts

@@ -1,8 +1,8 @@
 import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
 import BuildParameters from '../../../build-parameters';
-import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
+import { CommandHookService } from '../../services/hooks/command-hook-service';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import CloudRunner from '../../cloud-runner';
 
 class KubernetesJobSpecFactory {
@@ -19,63 +19,8 @@ class KubernetesJobSpecFactory {
     pvcName: string,
     jobName: string,
     k8s: any,
+    containerName: string,
   ) {
-    environment.push(
-      ...[
-        {
-          name: 'GITHUB_SHA',
-          value: buildGuid,
-        },
-        {
-          name: 'GITHUB_WORKSPACE',
-          value: '/data/repo',
-        },
-        {
-          name: 'PROJECT_PATH',
-          value: buildParameters.projectPath,
-        },
-        {
-          name: 'BUILD_PATH',
-          value: buildParameters.buildPath,
-        },
-        {
-          name: 'BUILD_FILE',
-          value: buildParameters.buildFile,
-        },
-        {
-          name: 'BUILD_NAME',
-          value: buildParameters.buildName,
-        },
-        {
-          name: 'BUILD_METHOD',
-          value: buildParameters.buildMethod,
-        },
-        {
-          name: 'CUSTOM_PARAMETERS',
-          value: buildParameters.customParameters,
-        },
-        {
-          name: 'CHOWN_FILES_TO',
-          value: buildParameters.chownFilesTo,
-        },
-        {
-          name: 'BUILD_TARGET',
-          value: buildParameters.targetPlatform,
-        },
-        {
-          name: 'ANDROID_VERSION_CODE',
-          value: buildParameters.androidVersionCode.toString(),
-        },
-        {
-          name: 'ANDROID_KEYSTORE_NAME',
-          value: buildParameters.androidKeystoreName,
-        },
-        {
-          name: 'ANDROID_KEYALIAS_NAME',
-          value: buildParameters.androidKeyaliasName,
-        },
-      ],
-    );
     const job = new k8s.V1Job();
     job.apiVersion = 'batch/v1';
     job.kind = 'Job';
@@ -87,6 +32,7 @@ class KubernetesJobSpecFactory {
       },
     };
     job.spec = {
+      ttlSecondsAfterFinished: 9999,
       backoffLimit: 0,
       template: {
         spec: {
@@ -100,16 +46,20 @@ class KubernetesJobSpecFactory {
           ],
           containers: [
             {
-              name: 'main',
+              ttlSecondsAfterFinished: 9999,
+              name: containerName,
               image,
               command: ['/bin/sh'],
-              args: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(command, CloudRunner.buildParameters)],
+              args: [
+                '-c',
+                `${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
+              ],
 
               workingDir: `${workingDirectory}`,
               resources: {
                 requests: {
-                  memory: buildParameters.cloudRunnerMemory || '750M',
+                  memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
-                  cpu: buildParameters.cloudRunnerCpu || '1',
+                  cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
                 },
               },
               env: [
@@ -135,7 +85,7 @@ class KubernetesJobSpecFactory {
               volumeMounts: [
                 {
                   name: 'build-mount',
-                  mountPath: `/${mountdir}`,
+                  mountPath: `${mountdir}`,
                 },
               ],
               lifecycle: {
@@ -158,7 +108,7 @@ class KubernetesJobSpecFactory {
       },
     };
 
-    job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '5Gi';
+    job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';
 
     return job;
   }

src/model/cloud-runner/providers/k8s/kubernetes-pods.ts

@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { CoreV1Api } from '@kubernetes/client-node';
 class KubernetesPods {
   public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
@@ -12,6 +12,12 @@ class KubernetesPods {
 
     return running;
   }
+  public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
+    const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
+    const phase = pods?.status?.phase || 'undefined status';
+
+    return phase;
+  }
 }
 
 export default KubernetesPods;

src/model/cloud-runner/providers/k8s/kubernetes-secret.ts

@@ -1,7 +1,7 @@
 import { CoreV1Api } from '@kubernetes/client-node';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import * as k8s from '@kubernetes/client-node';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import * as base64 from 'base-64';
 
 class KubernetesSecret {

src/model/cloud-runner/providers/k8s/kubernetes-storage.ts

@@ -2,7 +2,7 @@ import waitUntil from 'async-wait-until';
 import * as core from '@actions/core';
 import * as k8s from '@kubernetes/client-node';
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { IncomingMessage } from 'node:http';
 import GitHub from '../../../github';
 

src/model/cloud-runner/providers/k8s/kubernetes-task-runner.ts

@@ -1,12 +1,15 @@
-import { CoreV1Api, KubeConfig, Log } from '@kubernetes/client-node';
+import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
-import { Writable } from 'stream';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
-import * as core from '@actions/core';
-import { CloudRunnerStatics } from '../../cloud-runner-statics';
 import waitUntil from 'async-wait-until';
-import { FollowLogStreamService } from '../../services/follow-log-stream-service';
+import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
+import CloudRunner from '../../cloud-runner';
+import KubernetesPods from './kubernetes-pods';
+import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
 
 class KubernetesTaskRunner {
+  static lastReceivedTimestamp: number = 0;
+  static readonly maxRetry: number = 3;
+  static lastReceivedMessage: string = ``;
   static async runTask(
     kubeConfig: KubeConfig,
     kubeClient: CoreV1Api,
@@ -15,84 +18,120 @@ class KubernetesTaskRunner {
     containerName: string,
     namespace: string,
   ) {
-    CloudRunnerLogger.log(`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace}`);
-    const stream = new Writable();
     let output = '';
-    let didStreamAnyLogs: boolean = false;
     let shouldReadLogs = true;
     let shouldCleanup = true;
-    stream._write = (chunk, encoding, next) => {
+    let sinceTime = ``;
-      didStreamAnyLogs = true;
+    let retriesAfterFinish = 0;
-      let message = chunk.toString().trimRight(`\n`);
+    // eslint-disable-next-line no-constant-condition
-      message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
+    while (true) {
-      ({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
+      await new Promise((resolve) => setTimeout(resolve, 3000));
-        message,
+      const lastReceivedMessage =
-        shouldReadLogs,
+        KubernetesTaskRunner.lastReceivedTimestamp > 0
-        shouldCleanup,
+          ? `\nLast Log Message "${this.lastReceivedMessage}" ${this.lastReceivedTimestamp}`
-        output,
+          : ``;
-      ));
+      CloudRunnerLogger.log(
-      next();
+        `Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}\n${lastReceivedMessage}`,
-    };
-    const logOptions = {
-      follow: true,
-      pretty: false,
-      previous: false,
-    };
-    try {
-      const resultError = await new Promise((resolve) =>
-        new Log(kubeConfig).log(namespace, podName, containerName, stream, resolve, logOptions),
       );
-      stream.destroy();
+      if (KubernetesTaskRunner.lastReceivedTimestamp > 0) {
-      if (resultError) {
+        const currentDate = new Date(KubernetesTaskRunner.lastReceivedTimestamp);
-        throw resultError;
+        const dateTimeIsoString = currentDate.toISOString();
+        sinceTime = ` --since-time="${dateTimeIsoString}"`;
       }
-      if (!didStreamAnyLogs) {
+      let extraFlags = ``;
-        core.error('Failed to stream any logs, listing namespace events, check for an error with the container');
+      extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
-        core.error(
+        ? ` -f -c ${containerName}`
-          JSON.stringify(
+        : ` --previous`;
-            {
+      let lastMessageSeenIncludedInChunk = false;
-              events: (await kubeClient.listNamespacedEvent(namespace)).body.items
+      let lastMessageSeen = false;
-                .filter((x) => {
+
-                  return x.involvedObject.name === podName || x.involvedObject.name === jobName;
+      let logs;
-                })
+
-                .map((x) => {
+      try {
-                  return {
+        logs = await CloudRunnerSystem.Run(
-                    type: x.involvedObject.kind,
+          `kubectl logs ${podName}${extraFlags} --timestamps${sinceTime}`,
-                    name: x.involvedObject.name,
+          false,
-                    message: x.message,
+          true,
-                  };
-                }),
-            },
-            undefined,
-            4,
-          ),
         );
-        throw new Error(`No logs streamed from k8s`);
+      } catch (error: any) {
+        await new Promise((resolve) => setTimeout(resolve, 3000));
+        const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
+        CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
+        if (continueStreaming) {
+          continue;
+        }
+        if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
+          retriesAfterFinish++;
+
+          continue;
+        }
+        throw error;
       }
-    } catch (error) {
+      const splitLogs = logs.split(`\n`);
-      if (stream) {
+      for (const chunk of splitLogs) {
-        stream.destroy();
+        if (
+          chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) &&
+          KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) !== ``
+        ) {
+          CloudRunnerLogger.log(`Previous log message found ${chunk}`);
+          lastMessageSeenIncludedInChunk = true;
+        }
+      }
+      for (const chunk of splitLogs) {
+        const newDate = Date.parse(`${chunk.toString().split(`Z `)[0]}Z`);
+        if (chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``)) {
+          lastMessageSeen = true;
+        }
+        if (lastMessageSeenIncludedInChunk && !lastMessageSeen) {
+          continue;
+        }
+        const message = CloudRunner.buildParameters.cloudRunnerDebug ? chunk : chunk.split(`Z `)[1];
+        KubernetesTaskRunner.lastReceivedMessage = chunk;
+        KubernetesTaskRunner.lastReceivedTimestamp = newDate;
+        ({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
+          message,
+          shouldReadLogs,
+          shouldCleanup,
+          output,
+        ));
+      }
+      if (FollowLogStreamService.DidReceiveEndOfTransmission) {
+        CloudRunnerLogger.log('end of log stream');
+        break;
       }
-      throw error;
     }
-    CloudRunnerLogger.log('end of log stream');
 
     return output;
   }
 
   static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
     let success: boolean = false;
+    let message = ``;
     CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
     await waitUntil(
       async () => {
         const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
         const phase = status?.body.status?.phase;
         success = phase === 'Running';
-        CloudRunnerLogger.log(
+        message = `Phase:${status.body.status?.phase} \n Reason:${
-          `${status.body.status?.phase} ${status.body.status?.conditions?.[0].reason || ''} ${
+          status.body.status?.conditions?.[0].reason || ''
-            status.body.status?.conditions?.[0].message || ''
+        } \n Message:${status.body.status?.conditions?.[0].message || ''}`;
-          }`,
+
-        );
+        // CloudRunnerLogger.log(
+        //   JSON.stringify(
+        //     (await kubeClient.listNamespacedEvent(namespace)).body.items
+        //       .map((x) => {
+        //         return {
+        //           message: x.message || ``,
+        //           name: x.metadata.name || ``,
+        //           reason: x.reason || ``,
+        //         };
+        //       })
+        //       .filter((x) => x.name.includes(podName)),
+        //     undefined,
+        //     4,
+        //   ),
+        // );
         if (success || phase !== 'Pending') return true;
 
         return false;
@@ -102,6 +141,9 @@ class KubernetesTaskRunner {
         intervalBetweenAttempts: 15000,
       },
     );
+    if (!success) {
+      CloudRunnerLogger.log(message);
+    }
 
     return success;
   }

src/model/cloud-runner/providers/local/index.ts

@@ -1,9 +1,9 @@
 import BuildParameters from '../../../build-parameters';
-import { CloudRunnerSystem } from '../../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { ProviderInterface } from '../provider-interface';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 

src/model/cloud-runner/providers/provider-interface.ts

@@ -1,6 +1,6 @@
 import BuildParameters from '../../build-parameters';
-import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
-import CloudRunnerSecret from '../services/cloud-runner-secret';
+import CloudRunnerSecret from '../options/cloud-runner-secret';
 import { ProviderResource } from './provider-resource';
 import { ProviderWorkflow } from './provider-workflow';
 

src/model/cloud-runner/providers/test/index.ts

@@ -1,8 +1,8 @@
 import BuildParameters from '../../../build-parameters';
-import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
-import CloudRunnerLogger from '../../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { ProviderInterface } from '../provider-interface';
-import CloudRunnerSecret from '../../services/cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
 

src/model/cloud-runner/remote-client/caching.ts

@@ -2,10 +2,10 @@ import { assert } from 'node:console';
 import fs from 'node:fs';
 import path from 'node:path';
 import CloudRunner from '../cloud-runner';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
-import { CloudRunnerSystem } from '../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
-import { LfsHashing } from '../services/lfs-hashing';
+import { LfsHashing } from '../services/utility/lfs-hashing';
 import { RemoteClientLogger } from './remote-client-logger';
 import { Cli } from '../../cli/cli';
 import { CliFunction } from '../../cli/cli-functions-repository';
@@ -44,20 +44,21 @@ export class Caching {
   }
 
   public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
+    CloudRunnerLogger.log(`Pushing to cache ${sourceFolder}`);
     cacheArtifactName = cacheArtifactName.replace(' ', '');
     const startPath = process.cwd();
     let compressionSuffix = '';
-    if (CloudRunner.buildParameters.useLz4Compression === true) {
+    if (CloudRunner.buildParameters.useCompressionStrategy === true) {
       compressionSuffix = `.lz4`;
     }
-    CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useLz4Compression} ${compressionSuffix}`);
+    CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
     try {
       if (!(await fileExists(cacheFolder))) {
         await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
       }
       process.chdir(path.resolve(sourceFolder, '..'));
 
-      if (CloudRunner.buildParameters.cloudRunnerDebug) {
+      if (CloudRunner.buildParameters.cloudRunnerDebug === true) {
         CloudRunnerLogger.log(
           `Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
             sourceFolder,
@@ -69,11 +70,6 @@ export class Caching {
         `There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
       );
 
-      if (CloudRunner.buildParameters.cloudRunnerDebug) {
-        // await CloudRunnerSystem.Run(`tree -L 2 ./..`);
-        // await CloudRunnerSystem.Run(`tree -L 2`);
-      }
-
       if (contents.length === 0) {
         CloudRunnerLogger.log(
           `Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
@@ -102,9 +98,15 @@ export class Caching {
     process.chdir(`${startPath}`);
   }
   public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
+    CloudRunnerLogger.log(`Pulling from cache ${destinationFolder} ${CloudRunner.buildParameters.skipCache}`);
+    if (`${CloudRunner.buildParameters.skipCache}` === `true`) {
+      CloudRunnerLogger.log(`Skipping cache debugSkipCache is true`);
+
+      return;
+    }
     cacheArtifactName = cacheArtifactName.replace(' ', '');
     let compressionSuffix = '';
-    if (CloudRunner.buildParameters.useLz4Compression === true) {
+    if (CloudRunner.buildParameters.useCompressionStrategy === true) {
       compressionSuffix = `.lz4`;
     }
     const startPath = process.cwd();
@@ -160,7 +162,6 @@ export class Caching {
           RemoteClientLogger.logWarning(
             `cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
           );
-          await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
           throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
         }
       }

src/model/cloud-runner/remote-client/index.ts

@@ -1,56 +1,80 @@
 import fs from 'node:fs';
 import CloudRunner from '../cloud-runner';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
 import { Caching } from './caching';
-import { LfsHashing } from '../services/lfs-hashing';
+import { LfsHashing } from '../services/utility/lfs-hashing';
 import { RemoteClientLogger } from './remote-client-logger';
 import path from 'node:path';
 import { assert } from 'node:console';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
 import { CliFunction } from '../../cli/cli-functions-repository';
-import { CloudRunnerSystem } from '../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
 import YAML from 'yaml';
+import GitHub from '../../github';
+import BuildParameters from '../../build-parameters';
 
 export class RemoteClient {
-  public static async bootstrapRepository() {
+  @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
-    try {
+  static async runRemoteClientJob() {
-      await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
+    CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
-      await CloudRunnerSystem.Run(
+    if (!(await RemoteClient.handleRetainedWorkspace())) {
-        `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
+      await RemoteClient.bootstrapRepository();
-      );
-      process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
-      await RemoteClient.cloneRepoWithoutLFSFiles();
-      RemoteClient.replaceLargePackageReferencesWithSharedReferences();
-      await RemoteClient.sizeOfFolder(
-        'repo before lfs cache pull',
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
-      );
-      const lfsHashes = await LfsHashing.createLFSHashFiles();
-      if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
-        RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
-      }
-      await Caching.PullFromCache(
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
-        `${lfsHashes.lfsGuidSum}`,
-      );
-      await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
-      await RemoteClient.pullLatestLFS();
-      await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
-      await Caching.PushToCache(
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
-        `${lfsHashes.lfsGuidSum}`,
-      );
-      await Caching.PullFromCache(
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
-        CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
-      );
-      await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
-      await Caching.handleCachePurging();
-    } catch (error) {
-      throw error;
     }
+    await RemoteClient.runCustomHookFiles(`before-build`);
+  }
+  static async runCustomHookFiles(hookLifecycle: string) {
+    RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
+    const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
+    try {
+      const files = fs.readdirSync(gameCiCustomHooksPath);
+      for (const file of files) {
+        const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
+        const fileContentsObject = YAML.parse(fileContents.toString());
+        if (fileContentsObject.hook === hookLifecycle) {
+          RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
+          await CloudRunnerSystem.Run(fileContentsObject.commands);
+        }
+      }
+    } catch (error) {
+      RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
+    }
+  }
+  public static async bootstrapRepository() {
+    await CloudRunnerSystem.Run(
+      `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
+    );
+    await CloudRunnerSystem.Run(
+      `mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
+    );
+    await RemoteClient.cloneRepoWithoutLFSFiles();
+    await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
+    await RemoteClient.sizeOfFolder(
+      'repo before lfs cache pull',
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
+    );
+    const lfsHashes = await LfsHashing.createLFSHashFiles();
+    if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
+      RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
+    }
+    await Caching.PullFromCache(
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
+      `${lfsHashes.lfsGuidSum}`,
+    );
+    await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
+    await RemoteClient.pullLatestLFS();
+    await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
+    await Caching.PushToCache(
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
+      `${lfsHashes.lfsGuidSum}`,
+    );
+    await Caching.PullFromCache(
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
+      CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
+    );
+    await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
+    await Caching.handleCachePurging();
   }
 
   private static async sizeOfFolder(message: string, folder: string) {
@@ -62,58 +86,68 @@ export class RemoteClient {
 
   private static async cloneRepoWithoutLFSFiles() {
     process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
+    if (
+      fs.existsSync(CloudRunnerFolders.repoPathAbsolute) &&
+      !fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
+    ) {
+      await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.repoPathAbsolute}`);
+      CloudRunnerLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
+    }
 
     if (
-      CloudRunner.buildParameters.retainWorkspace &&
+      BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
       fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
     ) {
       process.chdir(CloudRunnerFolders.repoPathAbsolute);
       RemoteClientLogger.log(
-        `${CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${CloudRunner.buildParameters.retainWorkspace}`,
+        `${
+          CloudRunnerFolders.repoPathAbsolute
+        } repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
+          CloudRunner.buildParameters,
+        )}`,
       );
       await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`);
 
       return;
     }
 
-    if (fs.existsSync(CloudRunnerFolders.repoPathAbsolute)) {
+    RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
-      RemoteClientLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists cleaning up`);
+    await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
-      await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
+    RemoteClientLogger.log(`Cloning the repository being built:`);
-    }
+    await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
-
+    await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
     try {
-      RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
-      await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
-      RemoteClientLogger.log(`Cloning the repository being built:`);
-      await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
-      await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
       await CloudRunnerSystem.Run(
-        `git clone -q ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
+        `git clone ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
       );
-      process.chdir(CloudRunnerFolders.repoPathAbsolute);
+    } catch (error: any) {
-      await CloudRunnerSystem.Run(`git lfs install`);
-      assert(fs.existsSync(`.git`), 'git folder exists');
-      RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
-      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
-      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
-      assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
-      RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
-    } catch (error) {
-      await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
       throw error;
     }
+    process.chdir(CloudRunnerFolders.repoPathAbsolute);
+    await CloudRunnerSystem.Run(`git lfs install`);
+    assert(fs.existsSync(`.git`), 'git folder exists');
+    RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
+    if (CloudRunner.buildParameters.gitSha !== undefined) {
+      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
+    } else {
+      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
+      RemoteClientLogger.log(`buildParameter Git Sha is empty`);
+    }
+
+    assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
+    RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
   }
 
-  static replaceLargePackageReferencesWithSharedReferences() {
+  static async replaceLargePackageReferencesWithSharedReferences() {
-    if (CloudRunner.buildParameters.useSharedLargePackages) {
+    CloudRunnerLogger.log(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`);
+    GitHub.updateGitHubCheck(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`, ``);
+    if (CloudRunner.buildParameters.useLargePackages) {
       const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
       let manifest = fs.readFileSync(filePath, 'utf8');
       manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
       fs.writeFileSync(filePath, manifest);
-      if (CloudRunner.buildParameters.cloudRunnerDebug) {
+      CloudRunnerLogger.log(`Package Manifest \n ${manifest}`);
-        CloudRunnerLogger.log(`Package Manifest`);
+      GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
-        CloudRunnerLogger.log(manifest);
-      }
     }
   }
 
@@ -121,41 +155,31 @@ export class RemoteClient {
     process.chdir(CloudRunnerFolders.repoPathAbsolute);
     await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
     await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
-    await CloudRunnerSystem.Run(`git lfs pull`);
+    if (!CloudRunner.buildParameters.skipLfs) {
-    RemoteClientLogger.log(`pulled latest LFS files`);
+      await CloudRunnerSystem.Run(`git lfs pull`);
-    assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
+      RemoteClientLogger.log(`pulled latest LFS files`);
-  }
+      assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
-
-  @CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
-  static async runRemoteClientJob() {
-    // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
-    RemoteClient.handleRetainedWorkspace();
-
-    // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
-    await RemoteClient.bootstrapRepository();
-
-    // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
-    await RemoteClient.runCustomHookFiles(`before-build`);
-
-    // await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
-  }
-  static async runCustomHookFiles(hookLifecycle: string) {
-    RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
-    const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
-    const files = fs.readdirSync(gameCiCustomHooksPath);
-    for (const file of files) {
-      const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
-      const fileContentsObject = YAML.parse(fileContents.toString());
-      if (fileContentsObject.hook === hookLifecycle) {
-        RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
-        await CloudRunnerSystem.Run(fileContentsObject.commands);
-      }
     }
   }
-  static handleRetainedWorkspace() {
+  static async handleRetainedWorkspace() {
-    if (!CloudRunner.buildParameters.retainWorkspace) {
+    RemoteClientLogger.log(
-      return;
+      `Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)}`,
+    );
+    if (
+      BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
+      fs.existsSync(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)) &&
+      fs.existsSync(CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)))
+    ) {
+      CloudRunnerLogger.log(`Retained Workspace Already Exists!`);
+      process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
+      await CloudRunnerSystem.Run(`git fetch`);
+      await CloudRunnerSystem.Run(`git lfs pull`);
+      await CloudRunnerSystem.Run(`git reset --hard "${CloudRunner.buildParameters.gitSha}"`);
+      await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
+
+      return true;
     }
-    RemoteClientLogger.log(`Retained Workspace: ${CloudRunner.lockedWorkspace}`);
+
+    return false;
   }
 }

src/model/cloud-runner/remote-client/remote-client-logger.ts

@@ -1,4 +1,4 @@
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
 
 export class RemoteClientLogger {
   public static log(message: string) {

src/model/cloud-runner/services/cloud-runner-custom-hooks.ts

@@ -1,114 +0,0 @@
-import { BuildParameters, Input } from '../..';
-import YAML from 'yaml';
-import CloudRunnerSecret from './cloud-runner-secret';
-import { RemoteClientLogger } from '../remote-client/remote-client-logger';
-import path from 'node:path';
-import CloudRunnerOptions from '../cloud-runner-options';
-import fs from 'node:fs';
-
-// import CloudRunnerLogger from './cloud-runner-logger';
-
-export class CloudRunnerCustomHooks {
-  // TODO also accept hooks as yaml files in the repo
-  public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
-    const hooks = CloudRunnerCustomHooks.getHooks(buildParameters.customJobHooks).filter((x) => x.step.includes(`all`));
-
-    return `echo "---"
-      echo "start cloud runner init"
-      ${CloudRunnerOptions.cloudRunnerDebugEnv ? `printenv` : `#`}
-      echo "start of cloud runner job"
-      ${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
-      ${commands}
-      ${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
-      echo "end of cloud runner job"
-      echo "---${buildParameters.logId}"`;
-  }
-
-  public static getHooks(customJobHooks: string): Hook[] {
-    const experimentHooks = customJobHooks;
-    let output = new Array<Hook>();
-    if (experimentHooks && experimentHooks !== '') {
-      output = YAML.parse(experimentHooks);
-    }
-
-    return output.filter((x) => x.step !== undefined && x.hook !== undefined && x.hook.length > 0);
-  }
-
-  static GetCustomHooksFromFiles(hookLifecycle: string): Hook[] {
-    const results: Hook[] = [];
-    RemoteClientLogger.log(`GetCustomStepFiles: ${hookLifecycle}`);
-    try {
-      const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `hooks`);
-      const files = fs.readdirSync(gameCiCustomStepsPath);
-      for (const file of files) {
-        if (!CloudRunnerOptions.customHookFiles.includes(file.replace(`.yaml`, ``))) {
-          continue;
-        }
-        const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
-        const fileContentsObject = CloudRunnerCustomHooks.ParseHooks(fileContents)[0];
-        if (fileContentsObject.hook.includes(hookLifecycle)) {
-          results.push(fileContentsObject);
-        }
-      }
-    } catch (error) {
-      RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
-    }
-    RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
-
-    return results;
-  }
-
-  private static ConvertYamlSecrets(object: Hook) {
-    if (object.secrets === undefined) {
-      object.secrets = [];
-
-      return;
-    }
-    object.secrets = object.secrets.map((x) => {
-      return {
-        ParameterKey: x.ParameterKey,
-        EnvironmentVariable: Input.ToEnvVarFormat(x.ParameterKey),
-        ParameterValue: x.ParameterValue,
-      };
-    });
-  }
-
-  public static ParseHooks(steps: string): Hook[] {
-    if (steps === '') {
-      return [];
-    }
-
-    // if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
-
-    // CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
-
-    // }
-    const isArray = steps.replace(/\s/g, ``)[0] === `-`;
-    const object: Hook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
-    for (const hook of object) {
-      CloudRunnerCustomHooks.ConvertYamlSecrets(hook);
-      if (hook.secrets === undefined) {
-        hook.secrets = [];
-      }
-    }
-    if (object === undefined) {
-      throw new Error(`Failed to parse ${steps}`);
-    }
-
-    return object;
-  }
-
-  public static getSecrets(hooks: Hook[]) {
-    const secrets = hooks.map((x) => x.secrets).filter((x) => x !== undefined && x.length > 0);
-
-    // eslint-disable-next-line unicorn/no-array-reduce
-    return secrets.length > 0 ? secrets.reduce((x, y) => [...x, ...y]) : [];
-  }
-}
-export class Hook {
-  public commands!: string[];
-  public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
-  public name!: string;
-  public hook!: string[];
-  public step!: string[];
-}

src/model/cloud-runner/services/core/cloud-runner-system.ts

@@ -1,5 +1,5 @@
 import { exec } from 'child_process';
-import { RemoteClientLogger } from '../remote-client/remote-client-logger';
+import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
 
 export class CloudRunnerSystem {
   public static async RunAndReadLines(command: string): Promise<string[]> {

src/model/cloud-runner/services/core/follow-log-stream-service.ts

@@ -0,0 +1,57 @@
+import GitHub from '../../../github';
+import CloudRunner from '../../cloud-runner';
+import { CloudRunnerStatics } from '../../options/cloud-runner-statics';
+import CloudRunnerLogger from './cloud-runner-logger';
+import * as core from '@actions/core';
+
+export class FollowLogStreamService {
+  static Reset() {
+    FollowLogStreamService.DidReceiveEndOfTransmission = false;
+  }
+  static errors = ``;
+  public static DidReceiveEndOfTransmission = false;
+  public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
+    if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
+      CloudRunnerLogger.log('End of log transmission received');
+      FollowLogStreamService.DidReceiveEndOfTransmission = true;
+      shouldReadLogs = false;
+    } else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
+      GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
+      core.warning('LIBRARY NOT FOUND!');
+      core.setOutput('library-found', 'false');
+    } else if (message.includes('Build succeeded')) {
+      GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
+      core.setOutput('build-result', 'success');
+    } else if (message.includes('Build fail')) {
+      GitHub.updateGitHubCheck(
+        `Build failed\n${FollowLogStreamService.errors}`,
+        `Build failed`,
+        `failure`,
+        `completed`,
+      );
+      core.setOutput('build-result', 'failed');
+      core.setFailed('unity build failed');
+      core.error('BUILD FAILED!');
+    } else if (message.toLowerCase().includes('error ')) {
+      core.error(message);
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('error: ')) {
+      core.error(message);
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('command failed: ')) {
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('invalid ')) {
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('incompatible  ')) {
+      FollowLogStreamService.errors += `\n${message}`;
+    } else if (message.toLowerCase().includes('cannot be found')) {
+      FollowLogStreamService.errors += `\n${message}`;
+    }
+    if (CloudRunner.buildParameters.cloudRunnerDebug) {
+      output += `${message}\n`;
+    }
+    CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
+
+    return { shouldReadLogs, shouldCleanup, output };
+  }
+}

src/model/cloud-runner/services/core/shared-workspace-locking.ts

@@ -1,18 +1,17 @@
 import { CloudRunnerSystem } from './cloud-runner-system';
 import fs from 'node:fs';
 import CloudRunnerLogger from './cloud-runner-logger';
-import CloudRunnerOptions from '../cloud-runner-options';
+import BuildParameters from '../../../build-parameters';
-import BuildParameters from '../../build-parameters';
+import CloudRunner from '../../cloud-runner';
-import CloudRunner from '../cloud-runner';
 export class SharedWorkspaceLocking {
-  private static get workspaceBucketRoot() {
+  public static get workspaceBucketRoot() {
-    return `s3://${CloudRunner.buildParameters.awsBaseStackName}/`;
+    return `s3://${CloudRunner.buildParameters.awsStackName}/`;
   }
-  private static get workspaceRoot() {
+  public static get workspaceRoot() {
     return `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
   }
   public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
-    if (!(await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext))) {
+    if (!(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext))) {
       return [];
     }
 
@@ -20,79 +19,95 @@ export class SharedWorkspaceLocking {
       await SharedWorkspaceLocking.ReadLines(
         `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
       )
-    ).map((x) => x.replace(`/`, ``));
+    )
-  }
-  public static async DoesWorkspaceTopLevelExist(buildParametersContext: BuildParameters) {
-    await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`);
-
-    return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`))
       .map((x) => x.replace(`/`, ``))
-      .includes(buildParametersContext.cacheKey);
+      .filter((x) => x.endsWith(`_workspace`))
+      .map((x) => x.split(`_`)[1]);
   }
-  public static async GetAllLocks(workspace: string, buildParametersContext: BuildParameters): Promise<string[]> {
+  public static async DoesCacheKeyTopLevelExist(buildParametersContext: BuildParameters) {
+    try {
+      const rootLines = await SharedWorkspaceLocking.ReadLines(
+        `aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`,
+      );
+      const lockFolderExists = rootLines.map((x) => x.replace(`/`, ``)).includes(`locks`);
+
+      if (lockFolderExists) {
+        const lines = await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`);
+
+        return lines.map((x) => x.replace(`/`, ``)).includes(buildParametersContext.cacheKey);
+      } else {
+        return false;
+      }
+    } catch {
+      return false;
+    }
+  }
+
+  public static NewWorkspaceName() {
+    return `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
+  }
+  public static async GetAllLocksForWorkspace(
+    workspace: string,
+    buildParametersContext: BuildParameters,
+  ): Promise<string[]> {
     if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
       return [];
     }
 
     return (
       await SharedWorkspaceLocking.ReadLines(
-        `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
+        `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
       )
     )
       .map((x) => x.replace(`/`, ``))
-      .filter((x) => x.includes(`_lock`));
+      .filter((x) => x.includes(workspace) && x.endsWith(`_lock`));
   }
-  public static async GetOrCreateLockedWorkspace(
+  public static async GetLockedWorkspace(workspace: string, runId: string, buildParametersContext: BuildParameters) {
-    workspace: string,
+    if (buildParametersContext.maxRetainedWorkspaces === 0) {
-    runId: string,
+      return false;
-    buildParametersContext: BuildParameters,
-  ) {
-    if (!CloudRunnerOptions.retainWorkspaces) {
-      return;
     }
 
-    try {
+    if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
-      if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
+      const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
-        const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
+      CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
+      for (const element of workspaces) {
+        const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
         CloudRunnerLogger.log(
-          `run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`,
+          `run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
         );
-        for (const element of workspaces) {
-          await new Promise((promise) => setTimeout(promise, 1000));
-          const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
-          CloudRunnerLogger.log(`run agent: ${runId} try lock workspace: ${element} result: ${lockResult}`);
 
-          if (lockResult) {
+        if (lockResult) {
-            CloudRunner.lockedWorkspace = element;
+          return true;
-
-            return true;
-          }
         }
       }
-    } catch {
-      return;
     }
 
-    const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext, runId);
+    if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
+      workspace = SharedWorkspaceLocking.NewWorkspaceName();
+      CloudRunner.lockedWorkspace = workspace;
+    }
+
+    const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
+    const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
     CloudRunnerLogger.log(
-      `run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult}`,
+      `run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
     );
 
-    return createResult;
+    return createResult && lockResult;
   }
 
   public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
-    return (await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).includes(workspace);
+    return (
+      (await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).filter((x) => x.includes(workspace))
+        .length > 0
+    );
   }
   public static async HasWorkspaceLock(
     workspace: string,
     runId: string,
     buildParametersContext: BuildParameters,
   ): Promise<boolean> {
-    if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
+    const locks = (await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext))
-      return false;
-    }
-    const locks = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
       .map((x) => {
         return {
           name: x,
@@ -115,14 +130,11 @@ export class SharedWorkspaceLocking {
     const result: string[] = [];
     const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
     for (const element of workspaces) {
-      await new Promise((promise) => setTimeout(promise, 1500));
       const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
       const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
+      CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
       if (!isLocked && isBelowMax) {
         result.push(element);
-        CloudRunnerLogger.log(`workspace ${element} is free`);
-      } else {
-        CloudRunnerLogger.log(`workspace ${element} is NOT free ${!isLocked} ${isBelowMax}`);
       }
     }
 
@@ -171,60 +183,49 @@ export class SharedWorkspaceLocking {
 
     return (
       await SharedWorkspaceLocking.ReadLines(
-        `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
+        `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
       )
     )
       .map((x) => x.replace(`/`, ``))
-      .filter((x) => x.includes(`_workspace`))
+      .filter((x) => x.includes(workspace) && x.endsWith(`_workspace`))
       .map((x) => Number(x))[0];
   }
 
   public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
     if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
-      return false;
+      throw new Error(`workspace doesn't exist ${workspace}`);
     }
     const files = await SharedWorkspaceLocking.ReadLines(
-      `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
+      `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
     );
 
-    const workspaceFileDoesNotExists =
-      files.filter((x) => {
-        return x.includes(`_workspace`);
-      }).length === 0;
-
     const lockFilesExist =
       files.filter((x) => {
-        return x.includes(`_lock`);
+        return x.includes(workspace) && x.endsWith(`_lock`);
       }).length > 0;
 
-    return workspaceFileDoesNotExists || lockFilesExist;
+    return lockFilesExist;
   }
 
-  public static async CreateWorkspace(
+  public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
-    workspace: string,
+    if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
-    buildParametersContext: BuildParameters,
+      throw new Error(`${workspace} already exists`);
-    lockId: string = ``,
-  ): Promise<boolean> {
-    if (lockId !== ``) {
-      await SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext);
     }
     const timestamp = Date.now();
-    const file = `${timestamp}_workspace`;
+    const file = `${timestamp}_${workspace}_workspace`;
     fs.writeFileSync(file, '');
     await CloudRunnerSystem.Run(
-      `aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
+      `aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
       false,
       true,
     );
     fs.rmSync(file);
 
-    const workspaces = await SharedWorkspaceLocking.ReadLines(
+    const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
-      `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
-    );
 
     CloudRunnerLogger.log(`All workspaces ${workspaces}`);
     if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
-      CloudRunnerLogger.log(`Workspace is below max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
+      CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
       await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
 
       return false;
@@ -238,16 +239,30 @@ export class SharedWorkspaceLocking {
     runId: string,
     buildParametersContext: BuildParameters,
   ): Promise<boolean> {
-    const file = `${Date.now()}_${runId}_lock`;
+    const existingWorkspace = workspace.endsWith(`_workspace`);
+    const ending = existingWorkspace ? workspace : `${workspace}_workspace`;
+    const file = `${Date.now()}_${runId}_${ending}_lock`;
     fs.writeFileSync(file, '');
     await CloudRunnerSystem.Run(
-      `aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
+      `aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
       false,
       true,
     );
     fs.rmSync(file);
 
-    return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
+    const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
+
+    if (hasLock) {
+      CloudRunner.lockedWorkspace = workspace;
+    } else {
+      await CloudRunnerSystem.Run(
+        `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
+        false,
+        true,
+      );
+    }
+
+    return hasLock;
   }
 
   public static async ReleaseWorkspace(
@@ -255,31 +270,29 @@ export class SharedWorkspaceLocking {
     runId: string,
     buildParametersContext: BuildParameters,
   ): Promise<boolean> {
-    const file = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) =>
+    const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
-      x.includes(`_${runId}_lock`),
+    const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
-    );
+    CloudRunnerLogger.log(`All Locks ${files} ${workspace} ${runId}`);
     CloudRunnerLogger.log(`Deleting lock ${workspace}/${file}`);
-    CloudRunnerLogger.log(
+    CloudRunnerLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
-      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
-    );
     await CloudRunnerSystem.Run(
-      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
+      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
       false,
       true,
     );
 
-    return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
+    return !(await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext));
   }
 
   public static async CleanupWorkspace(workspace: string, buildParametersContext: BuildParameters) {
     await CloudRunnerSystem.Run(
-      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace} --recursive`,
+      `aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey} --exclude "*" --include "*_${workspace}_*"`,
       false,
       true,
     );
   }
 
-  private static async ReadLines(command: string): Promise<string[]> {
+  public static async ReadLines(command: string): Promise<string[]> {
     return CloudRunnerSystem.RunAndReadLines(command);
   }
 }

src/model/cloud-runner/services/core/task-parameter-serializer.ts

@@ -1,53 +1,50 @@
-import { Input } from '../..';
+import BuildParameters from '../../../build-parameters';
-import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
+import Input from '../../../input';
-import { CloudRunnerCustomHooks } from './cloud-runner-custom-hooks';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
-import CloudRunnerSecret from './cloud-runner-secret';
+import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
-import CloudRunnerQueryOverride from './cloud-runner-query-override';
+import CloudRunnerOptionsReader from '../../options/cloud-runner-options-reader';
-import CloudRunnerOptionsReader from './cloud-runner-options-reader';
+import CloudRunnerQueryOverride from '../../options/cloud-runner-query-override';
-import BuildParameters from '../../build-parameters';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
-import CloudRunnerOptions from '../cloud-runner-options';
+import { CommandHookService } from '../hooks/command-hook-service';
-import * as core from '@actions/core';
 
 export class TaskParameterSerializer {
-  static readonly blocked = new Set(['0', 'length', 'prototype', '', 'unityVersion']);
+  static readonly blockedParameterNames: Set<string> = new Set([
+    '0',
+    'length',
+    'prototype',
+    '',
+    'unityVersion',
+    'CACHE_UNITY_INSTALLATION_ON_MAC',
+    'RUNNER_TEMP_PATH',
+    'NAME',
+    'CUSTOM_JOB',
+  ]);
   public static createCloudRunnerEnvironmentVariables(
     buildParameters: BuildParameters,
   ): CloudRunnerEnvironmentVariable[] {
-    const result = this.uniqBy(
+    const result: CloudRunnerEnvironmentVariable[] = this.uniqBy(
       [
-        {
+        ...[
-          name: 'ContainerMemory',
+          { name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
-          value: buildParameters.cloudRunnerMemory,
+          { name: 'UNITY_VERSION', value: buildParameters.editorVersion },
-        },
+          { name: 'GITHUB_TOKEN', value: process.env.GITHUB_TOKEN },
-        {
+        ],
-          name: 'ContainerCpu',
-          value: buildParameters.cloudRunnerCpu,
-        },
-        {
-          name: 'BUILD_TARGET',
-          value: buildParameters.targetPlatform,
-        },
         ...TaskParameterSerializer.serializeFromObject(buildParameters),
-        ...TaskParameterSerializer.readInput(),
+        ...TaskParameterSerializer.serializeInput(),
-        ...CloudRunnerCustomHooks.getSecrets(CloudRunnerCustomHooks.getHooks(buildParameters.customJobHooks)),
+        ...TaskParameterSerializer.serializeCloudRunnerOptions(),
+        ...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
       ]
         .filter(
           (x) =>
-            !TaskParameterSerializer.blocked.has(x.name) &&
+            !TaskParameterSerializer.blockedParameterNames.has(x.name) &&
             x.value !== '' &&
             x.value !== undefined &&
-            x.name !== `CUSTOM_JOB` &&
-            x.name !== `GAMECI_CUSTOM_JOB` &&
             x.value !== `undefined`,
         )
         .map((x) => {
-          x.name = TaskParameterSerializer.ToEnvVarFormat(x.name);
+          x.name = `${TaskParameterSerializer.ToEnvVarFormat(x.name)}`;
           x.value = `${x.value}`;
 
-          if (buildParameters.cloudRunnerDebug && Number.isNaN(Number(x.name))) {
-            core.info(`[ERROR] found a number in task param serializer ${JSON.stringify(x)}`);
-          }
-
           return x;
         }),
       (item: CloudRunnerEnvironmentVariable) => item.name,
@@ -72,54 +69,58 @@ export class TaskParameterSerializer {
     const keys = [
       ...new Set(
         Object.getOwnPropertyNames(process.env)
-          .filter((x) => !this.blocked.has(x) && x.startsWith('GAMECI_'))
+          .filter((x) => !this.blockedParameterNames.has(x) && x.startsWith(''))
           .map((x) => TaskParameterSerializer.UndoEnvVarFormat(x)),
       ),
     ];
 
     for (const element of keys) {
       if (element !== `customJob`) {
-        buildParameters[element] = process.env[`GAMECI_${TaskParameterSerializer.ToEnvVarFormat(element)}`];
+        buildParameters[element] = process.env[`${TaskParameterSerializer.ToEnvVarFormat(element)}`];
       }
     }
 
     return buildParameters;
   }
 
-  private static readInput() {
+  private static serializeInput() {
     return TaskParameterSerializer.serializeFromType(Input);
   }
 
+  private static serializeCloudRunnerOptions() {
+    return TaskParameterSerializer.serializeFromType(CloudRunnerOptions);
+  }
+
   public static ToEnvVarFormat(input: string): string {
     return CloudRunnerOptions.ToEnvVarFormat(input);
   }
 
   public static UndoEnvVarFormat(element: string): string {
-    return this.camelize(element.replace('GAMECI_', '').toLowerCase().replace(/_+/g, ' '));
+    return this.camelize(element.toLowerCase().replace(/_+/g, ' '));
   }
 
   private static camelize(string: string) {
-    return string
+    return TaskParameterSerializer.uncapitalizeFirstLetter(
-      .replace(/(^\w)|([A-Z])|(\b\w)/g, function (word: string, index: number) {
+      string
-        return index === 0 ? word.toLowerCase() : word.toUpperCase();
+        .replace(/(^\w)|([A-Z])|(\b\w)/g, function (word: string, index: number) {
-      })
+          return index === 0 ? word.toLowerCase() : word.toUpperCase();
-      .replace(/\s+/g, '');
+        })
+        .replace(/\s+/g, ''),
+    );
+  }
+
+  private static uncapitalizeFirstLetter(string: string) {
+    return string.charAt(0).toLowerCase() + string.slice(1);
   }
 
   private static serializeFromObject(buildParameters: any) {
     const array: any[] = [];
-    const keys = Object.getOwnPropertyNames(buildParameters).filter((x) => !this.blocked.has(x));
+    const keys = Object.getOwnPropertyNames(buildParameters).filter((x) => !this.blockedParameterNames.has(x));
     for (const element of keys) {
-      array.push(
+      array.push({
-        {
+        name: TaskParameterSerializer.ToEnvVarFormat(element),
-          name: `GAMECI_${TaskParameterSerializer.ToEnvVarFormat(element)}`,
+        value: buildParameters[element],
-          value: buildParameters[element],
+      });
-        },
-        {
-          name: element,
-          value: buildParameters[element],
-        },
-      );
     }
 
     return array;

src/model/cloud-runner/services/follow-log-stream-service.ts

@@ -1,37 +0,0 @@
-import CloudRunnerLogger from './cloud-runner-logger';
-import * as core from '@actions/core';
-import CloudRunner from '../cloud-runner';
-import { CloudRunnerStatics } from '../cloud-runner-statics';
-import GitHub from '../../github';
-
-export class FollowLogStreamService {
-  public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
-    if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
-      CloudRunnerLogger.log('End of log transmission received');
-      shouldReadLogs = false;
-    } else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
-      GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
-      core.warning('LIBRARY NOT FOUND!');
-      core.setOutput('library-found', 'false');
-    } else if (message.includes('Build succeeded')) {
-      GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
-      core.setOutput('build-result', 'success');
-    } else if (message.includes('Build fail')) {
-      GitHub.updateGitHubCheck(`Build failed`, `Build failed`);
-      core.setOutput('build-result', 'failed');
-      core.setFailed('unity build failed');
-      core.error('BUILD FAILED!');
-    } else if (CloudRunner.buildParameters.cloudRunnerDebug && message.includes(': Listening for Jobs')) {
-      core.setOutput('cloud runner stop watching', 'true');
-      shouldReadLogs = false;
-      shouldCleanup = false;
-      core.warning('cloud runner stop watching');
-    }
-    if (CloudRunner.buildParameters.cloudRunnerDebug) {
-      output += `${message}\n`;
-    }
-    CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
-
-    return { shouldReadLogs, shouldCleanup, output };
-  }
-}

src/model/cloud-runner/services/hooks/command-hook-service.ts

@@ -0,0 +1,118 @@
+import { BuildParameters, Input } from '../../..';
+import YAML from 'yaml';
+import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
+import path from 'node:path';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
+import * as fs from 'node:fs';
+import CloudRunnerLogger from '../core/cloud-runner-logger';
+import { CommandHook } from './command-hook';
+
+// import CloudRunnerLogger from './cloud-runner-logger';
+
+export class CommandHookService {
+  public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
+    const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
+    CloudRunnerLogger.log(`Applying hooks ${hooks.length}`);
+
+    return `echo "---"
+echo "start cloud runner init"
+${CloudRunnerOptions.cloudRunnerDebug ? `printenv` : `#`}
+echo "start of cloud runner job"
+${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
+${commands}
+${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
+echo "end of cloud runner job"
+echo "---${buildParameters.logId}"`;
+  }
+
+  public static getHooks(customCommandHooks: string): CommandHook[] {
+    const experimentHooks = customCommandHooks;
+    let output = new Array<CommandHook>();
+    if (experimentHooks && experimentHooks !== '') {
+      try {
+        output = YAML.parse(experimentHooks);
+      } catch (error) {
+        throw error;
+      }
+    }
+
+    return [
+      ...output.filter((x) => x.hook !== undefined && x.hook.length > 0),
+      ...CommandHookService.GetCustomHooksFromFiles(`before`),
+      ...CommandHookService.GetCustomHooksFromFiles(`after`),
+    ];
+  }
+
+  static GetCustomHooksFromFiles(hookLifecycle: string): CommandHook[] {
+    const results: CommandHook[] = [];
+
+    // RemoteClientLogger.log(`GetCustomHookFiles: ${hookLifecycle}`);
+    try {
+      const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
+      const files = fs.readdirSync(gameCiCustomHooksPath);
+      for (const file of files) {
+        if (!CloudRunnerOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
+          continue;
+        }
+        const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
+        const fileContentsObject = CommandHookService.ParseHooks(fileContents)[0];
+        if (fileContentsObject.hook.includes(hookLifecycle)) {
+          results.push(fileContentsObject);
+        }
+      }
+    } catch (error) {
+      RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
+    }
+
+    // RemoteClientLogger.log(`Active Steps From Hooks: \n ${JSON.stringify(results, undefined, 4)}`);
+
+    return results;
+  }
+
+  private static ConvertYamlSecrets(object: CommandHook) {
+    if (object.secrets === undefined) {
+      object.secrets = [];
+
+      return;
+    }
+    object.secrets = object.secrets.map((x: any) => {
+      return {
+        ParameterKey: x.name,
+        EnvironmentVariable: Input.ToEnvVarFormat(x.name),
+        ParameterValue: x.value,
+      };
+    });
+  }
+
+  public static ParseHooks(hooks: string): CommandHook[] {
+    if (hooks === '') {
+      return [];
+    }
+
+    // if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
+
+    // CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
+
+    // }
+    const isArray = hooks.replace(/\s/g, ``)[0] === `-`;
+    const object: CommandHook[] = isArray ? YAML.parse(hooks) : [YAML.parse(hooks)];
+    for (const hook of object) {
+      CommandHookService.ConvertYamlSecrets(hook);
+      if (hook.secrets === undefined) {
+        hook.secrets = [];
+      }
+    }
+    if (object === undefined) {
+      throw new Error(`Failed to parse ${hooks}`);
+    }
+
+    return object;
+  }
+
+  public static getSecrets(hooks: any) {
+    const secrets = hooks.map((x: any) => x.secrets).filter((x: any) => x !== undefined && x.length > 0);
+
+    // eslint-disable-next-line unicorn/no-array-reduce
+    return secrets.length > 0 ? secrets.reduce((x: any, y: any) => [...x, ...y]) : [];
+  }
+}

src/model/cloud-runner/services/hooks/command-hook.ts

@@ -0,0 +1,9 @@
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
+
+export class CommandHook {
+  public commands: string[] = new Array<string>();
+  public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
+  public name!: string;
+  public hook!: string[];
+  public step!: string[];
+}

src/model/cloud-runner/services/hooks/container-hook-service.ts

@@ -1,32 +1,28 @@
 import YAML from 'yaml';
-import CloudRunner from '../cloud-runner';
+import CloudRunner from '../../cloud-runner';
 import * as core from '@actions/core';
-import { CustomWorkflow } from '../workflows/custom-workflow';
+import { CustomWorkflow } from '../../workflows/custom-workflow';
-import { RemoteClientLogger } from '../remote-client/remote-client-logger';
+import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
 import path from 'node:path';
 import fs from 'node:fs';
-import Input from '../../input';
+import Input from '../../../input';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
-import CloudRunnerLogger from './cloud-runner-logger';
+import { ContainerHook as ContainerHook } from './container-hook';
-import { CustomStep } from './custom-step';
+import { CloudRunnerStepParameters } from '../../options/cloud-runner-step-parameters';
-import { CloudRunnerStepState } from '../cloud-runner-step-state';
 
-export class CloudRunnerCustomSteps {
+export class ContainerHookService {
-  static GetCustomStepsFromFiles(hookLifecycle: string): CustomStep[] {
+  static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
-    const results: CustomStep[] = [];
+    const results: ContainerHook[] = [];
-    RemoteClientLogger.log(
-      `GetCustomStepFiles: ${hookLifecycle} CustomStepFiles: ${CloudRunnerOptions.customStepFiles}`,
-    );
     try {
-      const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `steps`);
+      const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
       const files = fs.readdirSync(gameCiCustomStepsPath);
       for (const file of files) {
-        if (!CloudRunnerOptions.customStepFiles.includes(file.replace(`.yaml`, ``))) {
+        if (!CloudRunnerOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
-          RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
+          // RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
           continue;
         }
         const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
-        const fileContentsObject = CloudRunnerCustomSteps.ParseSteps(fileContents)[0];
+        const fileContentsObject = ContainerHookService.ParseContainerHooks(fileContents)[0];
         if (fileContentsObject.hook === hookLifecycle) {
           results.push(fileContentsObject);
         }
@@ -34,9 +30,10 @@ export class CloudRunnerCustomSteps {
     } catch (error) {
       RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
     }
-    RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
 
-    const builtInCustomSteps: CustomStep[] = CloudRunnerCustomSteps.ParseSteps(
+    // RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
+
+    const builtInContainerHooks: ContainerHook[] = ContainerHookService.ParseContainerHooks(
       `- name: aws-s3-upload-build
   image: amazon/aws-cli
   hook: after
@@ -45,12 +42,12 @@ export class CloudRunnerCustomSteps {
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
     aws configure set region $AWS_DEFAULT_REGION --profile default
     aws s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
-      } s3://${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
+      } s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
       }
     rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
       }
   secrets:
   - name: awsAccessKeyId
@@ -65,19 +62,20 @@ export class CloudRunnerCustomSteps {
     aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
     aws configure set region $AWS_DEFAULT_REGION --profile default
-    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
+    aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
-    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build || true
+    aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build || true
+    mkdir -p /data/cache/$CACHE_KEY/build/
     aws s3 cp s3://${
-      CloudRunner.buildParameters.awsBaseStackName
+      CloudRunner.buildParameters.awsStackName
     }/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
       } /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
-        CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
+        CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
       }
   secrets:
-    - name: awsAccessKeyId
+    - name: AWS_ACCESS_KEY_ID
-    - name: awsSecretAccessKey
+    - name: AWS_SECRET_ACCESS_KEY
-    - name: awsDefaultRegion
+    - name: AWS_DEFAULT_REGION
     - name: BUILD_GUID_TARGET
 - name: steam-deploy-client
   image: steamcmd/steamcmd
@@ -123,19 +121,19 @@ export class CloudRunnerCustomSteps {
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
     aws configure set region $AWS_DEFAULT_REGION --profile default
     aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
-      CloudRunner.buildParameters.awsBaseStackName
+      CloudRunner.buildParameters.awsStackName
     }/cloud-runner-cache/$CACHE_KEY/lfs
     rm -r /data/cache/$CACHE_KEY/lfs
     aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
-      CloudRunner.buildParameters.awsBaseStackName
+      CloudRunner.buildParameters.awsStackName
     }/cloud-runner-cache/$CACHE_KEY/Library
     rm -r /data/cache/$CACHE_KEY/Library
   secrets:
-  - name: awsAccessKeyId
+  - name: AWS_ACCESS_KEY_ID
     value: ${process.env.AWS_ACCESS_KEY_ID || ``}
-  - name: awsSecretAccessKey
+  - name: AWS_SECRET_ACCESS_KEY
     value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
-  - name: awsDefaultRegion
+  - name: AWS_DEFAULT_REGION
     value: ${process.env.AWS_REGION || ``}
 - name: aws-s3-pull-cache
   image: amazon/aws-cli
@@ -144,30 +142,32 @@ export class CloudRunnerCustomSteps {
     aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
     aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
     aws configure set region $AWS_DEFAULT_REGION --profile default
-    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
+    mkdir -p /data/cache/$CACHE_KEY/Library/
-    aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/ || true
+    mkdir -p /data/cache/$CACHE_KEY/lfs/
-    BUCKET1="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
+    aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
+    aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/ || true
+    BUCKET1="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
     aws s3 ls $BUCKET1 || true
     OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
     aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
-    BUCKET2="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
+    BUCKET2="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
     aws s3 ls $BUCKET2 || true
     OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
     aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
   secrets:
-  - name: awsAccessKeyId
+  - name: AWS_ACCESS_KEY_ID
     value: ${process.env.AWS_ACCESS_KEY_ID || ``}
-  - name: awsSecretAccessKey
+  - name: AWS_SECRET_ACCESS_KEY
     value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
-  - name: awsDefaultRegion
+  - name: AWS_DEFAULT_REGION
     value: ${process.env.AWS_REGION || ``}
 - name: debug-cache
   image: ubuntu
   hook: after
   commands: |
     apt-get update > /dev/null
-    ${CloudRunnerOptions.cloudRunnerDebugTree ? `apt-get install -y tree > /dev/null` : `#`}
+    ${CloudRunnerOptions.cloudRunnerDebug ? `apt-get install -y tree > /dev/null` : `#`}
-    ${CloudRunnerOptions.cloudRunnerDebugTree ? `tree -L 3 /data/cache` : `#`}
+    ${CloudRunnerOptions.cloudRunnerDebug ? `tree -L 3 /data/cache` : `#`}
   secrets:
   - name: awsAccessKeyId
     value: ${process.env.AWS_ACCESS_KEY_ID || ``}
@@ -175,15 +175,15 @@ export class CloudRunnerCustomSteps {
     value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
   - name: awsDefaultRegion
     value: ${process.env.AWS_REGION || ``}`,
-    ).filter((x) => CloudRunnerOptions.customStepFiles.includes(x.name) && x.hook === hookLifecycle);
+    ).filter((x) => CloudRunnerOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
-    if (builtInCustomSteps.length > 0) {
+    if (builtInContainerHooks.length > 0) {
-      results.push(...builtInCustomSteps);
+      results.push(...builtInContainerHooks);
     }
 
     return results;
   }
 
-  private static ConvertYamlSecrets(object: CustomStep) {
+  private static ConvertYamlSecrets(object: ContainerHook) {
     if (object.secrets === undefined) {
       object.secrets = [];
 
@@ -198,21 +198,21 @@ export class CloudRunnerCustomSteps {
     });
   }
 
-  public static ParseSteps(steps: string): CustomStep[] {
+  public static ParseContainerHooks(steps: string): ContainerHook[] {
     if (steps === '') {
       return [];
     }
     const isArray = steps.replace(/\s/g, ``)[0] === `-`;
-    const object: CustomStep[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
+    const object: ContainerHook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
     for (const step of object) {
-      CloudRunnerCustomSteps.ConvertYamlSecrets(step);
+      ContainerHookService.ConvertYamlSecrets(step);
       if (step.secrets === undefined) {
         step.secrets = [];
       } else {
         for (const secret of step.secrets) {
           if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
             if (CloudRunner.buildParameters?.cloudRunnerDebug) {
-              CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
+              // CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
             }
             secret.ParameterValue = process.env[secret.ParameterKey] || ``;
           }
@@ -229,16 +229,16 @@ export class CloudRunnerCustomSteps {
     return object;
   }
 
-  static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepState) {
+  static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
     let output = ``;
-    const steps: CustomStep[] = [
+    const steps: ContainerHook[] = [
-      ...CloudRunnerCustomSteps.ParseSteps(CloudRunner.buildParameters.postBuildSteps),
+      ...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.postBuildContainerHooks),
-      ...CloudRunnerCustomSteps.GetCustomStepsFromFiles(`after`),
+      ...ContainerHookService.GetContainerHooksFromFiles(`after`),
     ];
 
     if (steps.length > 0) {
       if (!CloudRunner.buildParameters.isCliMode) core.startGroup('post build steps');
-      output += await CustomWorkflow.runCustomJob(
+      output += await CustomWorkflow.runContainerJob(
         steps,
         cloudRunnerStepState.environment,
         cloudRunnerStepState.secrets,
@@ -248,16 +248,16 @@ export class CloudRunnerCustomSteps {
 
     return output;
   }
-  static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepState) {
+  static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
     let output = ``;
-    const steps: CustomStep[] = [
+    const steps: ContainerHook[] = [
-      ...CloudRunnerCustomSteps.ParseSteps(CloudRunner.buildParameters.preBuildSteps),
+      ...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.preBuildContainerHooks),
-      ...CloudRunnerCustomSteps.GetCustomStepsFromFiles(`before`),
+      ...ContainerHookService.GetContainerHooksFromFiles(`before`),
     ];
 
     if (steps.length > 0) {
       if (!CloudRunner.buildParameters.isCliMode) core.startGroup('pre build steps');
-      output += await CustomWorkflow.runCustomJob(
+      output += await CustomWorkflow.runContainerJob(
         steps,
         cloudRunnerStepState.environment,
         cloudRunnerStepState.secrets,

src/model/cloud-runner/services/hooks/container-hook.ts

@@ -1,6 +1,6 @@
-import CloudRunnerSecret from './cloud-runner-secret';
+import CloudRunnerSecret from '../../options/cloud-runner-secret';
 
-export class CustomStep {
+export class ContainerHook {
   public commands!: string;
   public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
   public name!: string;

src/model/cloud-runner/services/lfs-hashing.ts

@@ -1,47 +0,0 @@
-import path from 'node:path';
-import { CloudRunnerFolders } from './cloud-runner-folders';
-import { CloudRunnerSystem } from './cloud-runner-system';
-import fs from 'node:fs';
-import { assert } from 'node:console';
-import { Cli } from '../../cli/cli';
-import { CliFunction } from '../../cli/cli-functions-repository';
-
-export class LfsHashing {
-  public static async createLFSHashFiles() {
-    try {
-      await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
-      await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
-      assert(fs.existsSync(`.lfs-assets-guid-sum`));
-      assert(fs.existsSync(`.lfs-assets-guid`));
-      const lfsHashes = {
-        lfsGuid: fs
-          .readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
-          .replace(/\n/g, ``),
-        lfsGuidSum: fs
-          .readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
-          .replace('  .lfs-assets-guid', '')
-          .replace(/\n/g, ``),
-      };
-
-      return lfsHashes;
-    } catch (error) {
-      throw error;
-    }
-  }
-  public static async hashAllFiles(folder: string) {
-    const startPath = process.cwd();
-    process.chdir(folder);
-    const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
-      .replace(/\n/g, '')
-      .split(` `)[0];
-    process.chdir(startPath);
-
-    return result;
-  }
-
-  @CliFunction(`hash`, `hash all folder contents`)
-  static async hash() {
-    const folder = Cli.options!['cachePushFrom'];
-    LfsHashing.hashAllFiles(folder);
-  }
-}

src/model/cloud-runner/services/utility/lfs-hashing.ts

@@ -0,0 +1,43 @@
+import path from 'node:path';
+import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
+import { CloudRunnerSystem } from '../core/cloud-runner-system';
+import fs from 'node:fs';
+import { Cli } from '../../../cli/cli';
+import { CliFunction } from '../../../cli/cli-functions-repository';
+
+export class LfsHashing {
+  public static async createLFSHashFiles() {
+    await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
+    await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
+    const lfsHashes = {
+      lfsGuid: fs
+        .readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
+        .replace(/\n/g, ``),
+      lfsGuidSum: fs
+        .readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
+        .replace('  .lfs-assets-guid', '')
+        .replace(/\n/g, ``),
+    };
+
+    return lfsHashes;
+  }
+  public static async hashAllFiles(folder: string) {
+    const startPath = process.cwd();
+    process.chdir(folder);
+    const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
+      .replace(/\n/g, '')
+      .split(` `)[0];
+    process.chdir(startPath);
+
+    return result;
+  }
+
+  @CliFunction(`hash`, `hash all folder contents`)
+  static async hash() {
+    if (!Cli.options) {
+      return;
+    }
+    const folder = Cli.options['cachePushFrom'];
+    LfsHashing.hashAllFiles(folder);
+  }
+}

src/model/cloud-runner/tests/cloud-runner-async-workflow.test.ts

@@ -2,7 +2,7 @@ import { BuildParameters, ImageTag } from '../..';
 import CloudRunner from '../cloud-runner';
 import UnityVersioning from '../../unity-versioning';
 import { Cli } from '../../cli/cli';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from '../options/cloud-runner-options';
 import setups from './cloud-runner-suite.test';
 import { OptionValues } from 'commander';
 
@@ -15,7 +15,7 @@ describe('Cloud Runner Async Workflows', () => {
   setups();
   it('Responds', () => {});
 
-  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `local-docker`) {
+  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
     it('Async Workflows', async () => {
       // Setup parameters
       const buildParameter = await CreateParameters({

src/model/cloud-runner/tests/cloud-runner-caching.test.ts

@@ -4,14 +4,15 @@ import BuildParameters from '../../build-parameters';
 import { Cli } from '../../cli/cli';
 import UnityVersioning from '../../unity-versioning';
 import CloudRunner from '../cloud-runner';
-import { CloudRunnerSystem } from '../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
 import { Caching } from '../remote-client/caching';
 import { v4 as uuidv4 } from 'uuid';
 import GitHub from '../../github';
+import CloudRunnerOptions from '../options/cloud-runner-options';
 describe('Cloud Runner (Remote Client) Caching', () => {
   it('responds', () => {});
-  if (process.platform === 'linux') {
+  if (CloudRunnerOptions.providerStrategy === `local-docker`) {
-    it.skip('Simple caching works', async () => {
+    it('Simple caching works', async () => {
       Cli.options = {
         versioning: 'None',
         projectPath: 'test-project',

src/model/cloud-runner/tests/cloud-runner-environment-serializer.test.ts

@@ -1,46 +0,0 @@
-import { BuildParameters } from '../..';
-import { TaskParameterSerializer } from '../services/task-parameter-serializer';
-import UnityVersioning from '../../unity-versioning';
-import { Cli } from '../../cli/cli';
-import GitHub from '../../github';
-import setups from './cloud-runner-suite.test';
-import { OptionValues } from 'commander';
-
-async function CreateParameters(overrides: OptionValues | undefined) {
-  if (overrides) {
-    Cli.options = overrides;
-  }
-  const originalValue = GitHub.githubInputEnabled;
-  GitHub.githubInputEnabled = false;
-  const results = await BuildParameters.create();
-  GitHub.githubInputEnabled = originalValue;
-  delete Cli.options;
-
-  return results;
-}
-describe('Cloud Runner Environment Serializer', () => {
-  setups();
-  const testSecretName = 'testSecretName';
-  const testSecretValue = 'testSecretValue';
-  it('Cloud Runner Parameter Serialization', async () => {
-    // Setup parameters
-    const buildParameter = await CreateParameters({
-      versioning: 'None',
-      projectPath: 'test-project',
-      unityVersion: UnityVersioning.read('test-project'),
-      customJob: `
-      - name: 'step 1'
-        image: 'alpine'
-        commands: 'printenv'
-        secrets:
-          - name: '${testSecretName}'
-            value: '${testSecretValue}'
-      `,
-    });
-
-    const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
-    expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
-    const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
-    expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
-  });
-});

src/model/cloud-runner/tests/cloud-runner-environment.test.ts

@@ -1,20 +1,26 @@
-import { BuildParameters, ImageTag } from '../..';
+import { BuildParameters, CloudRunner, ImageTag, Input } from '../..';
-import CloudRunner from '../cloud-runner';
+import { TaskParameterSerializer } from '../services/core/task-parameter-serializer';
-import Input from '../../input';
-import { CloudRunnerStatics } from '../cloud-runner-statics';
-import { TaskParameterSerializer } from '../services/task-parameter-serializer';
 import UnityVersioning from '../../unity-versioning';
 import { Cli } from '../../cli/cli';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import GitHub from '../../github';
-import CloudRunnerOptions from '../cloud-runner-options';
 import setups from './cloud-runner-suite.test';
-import { OptionValues } from 'commander';
+import { CloudRunnerStatics } from '../options/cloud-runner-statics';
+import CloudRunnerOptions from '../options/cloud-runner-options';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
 
-async function CreateParameters(overrides: OptionValues | undefined) {
+async function CreateParameters(overrides: any) {
-  if (overrides) Cli.options = overrides;
+  if (overrides) {
+    Cli.options = overrides;
+  }
+  const originalValue = GitHub.githubInputEnabled;
+  GitHub.githubInputEnabled = false;
+  const results = await BuildParameters.create();
+  GitHub.githubInputEnabled = originalValue;
+  delete Cli.options;
 
-  return BuildParameters.create();
+  return results;
 }
+
 describe('Cloud Runner Sync Environments', () => {
   setups();
   const testSecretName = 'testSecretName';
@@ -62,7 +68,7 @@ describe('Cloud Runner Sync Environments', () => {
           return x;
         })
         .filter((element) => {
-          return !['UNITY_LICENSE', 'CUSTOM_JOB'].includes(element.name);
+          return !['UNITY_LICENSE', 'UNITY_LICENSE', 'CUSTOM_JOB', 'CUSTOM_JOB'].includes(element.name);
         });
       const newLinePurgedFile = file
         .replace(/\s+/g, '')
@@ -76,3 +82,30 @@ describe('Cloud Runner Sync Environments', () => {
     }, 1_000_000_000);
   }
 });
+
+describe('Cloud Runner Environment Serializer', () => {
+  setups();
+  const testSecretName = 'testSecretName';
+  const testSecretValue = 'testSecretValue';
+  it('Cloud Runner Parameter Serialization', async () => {
+    // Setup parameters
+    const buildParameter = await CreateParameters({
+      versioning: 'None',
+      projectPath: 'test-project',
+      unityVersion: UnityVersioning.read('test-project'),
+      customJob: `
+      - name: 'step 1'
+        image: 'alpine'
+        commands: 'printenv'
+        secrets:
+          - name: '${testSecretName}'
+            value: '${testSecretValue}'
+      `,
+    });
+
+    const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
+    expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
+    const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
+    expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
+  });
+});

src/model/cloud-runner/tests/cloud-runner-hooks.test.ts

@@ -0,0 +1,114 @@
+import CloudRunner from '../cloud-runner';
+import { BuildParameters, ImageTag } from '../..';
+import UnityVersioning from '../../unity-versioning';
+import { Cli } from '../../cli/cli';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
+import { v4 as uuidv4 } from 'uuid';
+import CloudRunnerOptions from '../options/cloud-runner-options';
+import setups from './cloud-runner-suite.test';
+import { ContainerHookService } from '../services/hooks/container-hook-service';
+import { CommandHookService } from '../services/hooks/command-hook-service';
+
+async function CreateParameters(overrides: any) {
+  if (overrides) {
+    Cli.options = overrides;
+  }
+
+  return await BuildParameters.create();
+}
+
+describe('Cloud Runner Custom Hooks And Steps', () => {
+  it('Responds', () => {});
+  setups();
+  it('Check parsing and reading of steps', async () => {
+    const yamlString = `hook: before
+commands: echo "test"`;
+    const yamlString2 = `- hook: before
+  commands: echo "test"`;
+    const overrides = {
+      versioning: 'None',
+      projectPath: 'test-project',
+      unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+      targetPlatform: 'StandaloneLinux64',
+      cacheKey: `test-case-${uuidv4()}`,
+    };
+    CloudRunner.setup(await CreateParameters(overrides));
+    const stringObject = ContainerHookService.ParseContainerHooks(yamlString);
+    const stringObject2 = ContainerHookService.ParseContainerHooks(yamlString2);
+
+    CloudRunnerLogger.log(yamlString);
+    CloudRunnerLogger.log(JSON.stringify(stringObject, undefined, 4));
+
+    expect(stringObject.length).toBe(1);
+    expect(stringObject[0].hook).toBe(`before`);
+    expect(stringObject2.length).toBe(1);
+    expect(stringObject2[0].hook).toBe(`before`);
+
+    const getCustomStepsFromFiles = ContainerHookService.GetContainerHooksFromFiles(`before`);
+    CloudRunnerLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
+  });
+  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `k8s`) {
+    it('Should be 1 before and 1 after hook', async () => {
+      const overrides = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
+        commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
+      };
+      const buildParameter2 = await CreateParameters(overrides);
+      await CloudRunner.setup(buildParameter2);
+      const beforeHooks = CommandHookService.GetCustomHooksFromFiles(`before`);
+      const afterHooks = CommandHookService.GetCustomHooksFromFiles(`after`);
+      expect(beforeHooks).toHaveLength(1);
+      expect(afterHooks).toHaveLength(1);
+    });
+    it('Should be 1 before and 1 after step', async () => {
+      const overrides = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
+        commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
+      };
+      const buildParameter2 = await CreateParameters(overrides);
+      await CloudRunner.setup(buildParameter2);
+      const beforeSteps = ContainerHookService.GetContainerHooksFromFiles(`before`);
+      const afterSteps = ContainerHookService.GetContainerHooksFromFiles(`after`);
+      expect(beforeSteps).toHaveLength(1);
+      expect(afterSteps).toHaveLength(1);
+    });
+    it('Run build once - check for pre and post custom hooks run contents', async () => {
+      const overrides = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
+        commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
+      };
+      const buildParameter2 = await CreateParameters(overrides);
+      const baseImage2 = new ImageTag(buildParameter2);
+      const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
+      CloudRunnerLogger.log(`run 2 succeeded`);
+
+      const buildContainsBuildSucceeded = results2.includes('Build succeeded');
+      const buildContainsPreBuildHookRunMessage = results2.includes('before-build hook test!');
+      const buildContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
+
+      const buildContainsPreBuildStepMessage = results2.includes('before-build step test!');
+      const buildContainsPostBuildStepMessage = results2.includes('after-build step test!');
+
+      expect(buildContainsBuildSucceeded).toBeTruthy();
+      expect(buildContainsPreBuildHookRunMessage).toBeTruthy();
+      expect(buildContainsPostBuildHookRunMessage).toBeTruthy();
+      expect(buildContainsPreBuildStepMessage).toBeTruthy();
+      expect(buildContainsPostBuildStepMessage).toBeTruthy();
+    }, 1_000_000_000);
+  }
+});

src/model/cloud-runner/tests/cloud-runner-local-persistence.test.ts

@@ -0,0 +1,53 @@
+import { ImageTag } from '../..';
+import CloudRunner from '../cloud-runner';
+import UnityVersioning from '../../unity-versioning';
+import CloudRunnerOptions from '../options/cloud-runner-options';
+import setups from './cloud-runner-suite.test';
+import fs from 'node:fs';
+import { CreateParameters } from './create-test-parameter';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
+
+describe('Cloud Runner Local Docker Workflows', () => {
+  setups();
+  it('Responds', () => {});
+
+  if (CloudRunnerOptions.providerStrategy === `local-docker`) {
+    it('inspect stateful folder of workflows', async () => {
+      const testValue = `the state in a job exits in the expected local-docker folder`;
+
+      // Setup parameters
+      const buildParameter = await CreateParameters({
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.read('test-project'),
+        customJob: `
+        - name: 'step 1'
+          image: 'ubuntu'
+          commands: 'echo "${testValue}" >> /data/test-out-state.txt'
+        `,
+      });
+      const buildParameter2 = await CreateParameters({
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.read('test-project'),
+        customJob: `
+        - name: 'step 1'
+          image: 'ubuntu'
+          commands: 'cat /data/test-out-state.txt >> /data/test-out-state-2.txt'
+        `,
+      });
+      const baseImage = new ImageTag(buildParameter);
+
+      // Run the job
+      await CloudRunner.run(buildParameter, baseImage.toString());
+      await CloudRunner.run(buildParameter2, baseImage.toString());
+
+      const outputFile = fs.readFileSync(`./cloud-runner-cache/test-out-state.txt`, `utf-8`);
+      expect(outputFile).toMatch(testValue);
+
+      const outputFile2 = fs.readFileSync(`./cloud-runner-cache/test-out-state-2.txt`, `utf-8`);
+      expect(outputFile2).toMatch(testValue);
+      CloudRunnerLogger.log(outputFile);
+    }, 1_000_000_000);
+  }
+});

src/model/cloud-runner/tests/cloud-runner-locking-core.test.ts

@@ -0,0 +1,115 @@
+import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
+import { Cli } from '../../cli/cli';
+import setups from './cloud-runner-suite.test';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
+import { v4 as uuidv4 } from 'uuid';
+import CloudRunnerOptions from '../options/cloud-runner-options';
+import UnityVersioning from '../../unity-versioning';
+import BuildParameters from '../../build-parameters';
+import CloudRunner from '../cloud-runner';
+
+async function CreateParameters(overrides: any) {
+  if (overrides) {
+    Cli.options = overrides;
+  }
+
+  return await BuildParameters.create();
+}
+
+describe('Cloud Runner Locking Core', () => {
+  setups();
+  it('Responds', () => {});
+  if (CloudRunnerOptions.cloudRunnerDebug) {
+    it(`Create Workspace`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const buildParameters = await CreateParameters(overrides);
+      CloudRunner.buildParameters = buildParameters;
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
+    }, 150000);
+    it(`Create Workspace And Lock Workspace`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const runId = uuidv4();
+      const buildParameters = await CreateParameters(overrides);
+      CloudRunner.buildParameters = buildParameters;
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+    }, 150000);
+    it(`0 free workspaces after locking`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const buildParameters = await CreateParameters(overrides);
+
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      const runId = uuidv4();
+      CloudRunner.buildParameters = buildParameters;
+      expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters)).toHaveLength(1);
+      expect(await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters)).toHaveLength(1);
+      expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
+
+      const files = await SharedWorkspaceLocking.ReadLines(
+        `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParameters.cacheKey}/`,
+      );
+
+      const lockFilesExist =
+        files.filter((x) => {
+          return x.includes(newWorkspaceName) && x.endsWith(`_lock`);
+        }).length > 0;
+
+      expect(files).toHaveLength(2);
+      expect(
+        files.filter((x) => {
+          return x.includes(newWorkspaceName) && x.endsWith(`_lock`);
+        }),
+      ).toHaveLength(1);
+      expect(lockFilesExist).toBeTruthy();
+      const result: string[] = [];
+      const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters);
+      for (const element of workspaces) {
+        expect((await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters)).join()).toContain(element);
+        expect(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters)).toHaveLength(1);
+        expect(await SharedWorkspaceLocking.DoesWorkspaceExist(element, buildParameters)).toBeTruthy();
+        await new Promise((promise) => setTimeout(promise, 1500));
+        const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParameters);
+        const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParameters);
+        CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
+        const lock = files.find((x) => {
+          return x.endsWith(`_lock`);
+        });
+        expect(lock).toContain(element);
+        expect(isLocked).toBeTruthy();
+        expect(isBelowMax).toBeTruthy();
+        if (!isLocked && isBelowMax) {
+          result.push(element);
+        }
+      }
+      expect(result).toHaveLength(0);
+      expect(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters)).toHaveLength(0);
+    }, 300000);
+  }
+});

src/model/cloud-runner/tests/cloud-runner-locking-get-locked.test.ts

@@ -0,0 +1,156 @@
+import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
+import { Cli } from '../../cli/cli';
+import setups from './cloud-runner-suite.test';
+import { v4 as uuidv4 } from 'uuid';
+import CloudRunnerOptions from '../options/cloud-runner-options';
+import UnityVersioning from '../../unity-versioning';
+import BuildParameters from '../../build-parameters';
+import CloudRunner from '../cloud-runner';
+
+async function CreateParameters(overrides: any) {
+  if (overrides) {
+    Cli.options = overrides;
+  }
+
+  return await BuildParameters.create();
+}
+
+describe('Cloud Runner Locking Get Locked Workspace', () => {
+  setups();
+  it('Responds', () => {});
+  if (CloudRunnerOptions.cloudRunnerDebug) {
+    it(`Get locked workspace From No Workspace`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const buildParameters = await CreateParameters(overrides);
+
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      const runId = uuidv4();
+      CloudRunner.buildParameters = buildParameters;
+      expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+    }, 150000);
+    it(`Get locked workspace from unlocked`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const buildParameters = await CreateParameters(overrides);
+
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      const runId = uuidv4();
+      CloudRunner.buildParameters = buildParameters;
+      expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
+    }, 300000);
+    it(`Get locked workspace from locked`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const buildParameters = await CreateParameters(overrides);
+
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      const runId = uuidv4();
+      const runId2 = uuidv4();
+      CloudRunner.buildParameters = buildParameters;
+      expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.IsWorkspaceBelowMax(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
+      expect(CloudRunner.lockedWorkspace).not.toMatch(newWorkspaceName);
+    }, 300000);
+    it(`Get locked workspace after double lock and one unlock`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const buildParameters = await CreateParameters(overrides);
+
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      const runId = uuidv4();
+      const runId2 = uuidv4();
+      CloudRunner.buildParameters = buildParameters;
+      expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
+      expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
+      expect(CloudRunner.lockedWorkspace).not.toContain(newWorkspaceName);
+    }, 300000);
+    it(`Get locked workspace after double lock and unlock`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const buildParameters = await CreateParameters(overrides);
+
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      const runId = uuidv4();
+      const runId2 = uuidv4();
+      CloudRunner.buildParameters = buildParameters;
+      expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
+      expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
+      expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
+      expect(CloudRunner.lockedWorkspace).toContain(newWorkspaceName);
+    }, 300000);
+    it(`Get locked workspace from unlocked was locked`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const buildParameters = await CreateParameters(overrides);
+
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      const runId = uuidv4();
+      CloudRunner.buildParameters = buildParameters;
+      expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+      expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
+    }, 300000);
+  }
+});

src/model/cloud-runner/tests/cloud-runner-run-once-custom-hooks.test.ts

@@ -1,79 +0,0 @@
-import CloudRunner from '../cloud-runner';
-import { BuildParameters, ImageTag } from '../..';
-import UnityVersioning from '../../unity-versioning';
-import { Cli } from '../../cli/cli';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
-import { v4 as uuidv4 } from 'uuid';
-import CloudRunnerOptions from '../cloud-runner-options';
-import setups from './cloud-runner-suite.test';
-import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
-import { OptionValues } from 'commander';
-
-async function CreateParameters(overrides: OptionValues | undefined) {
-  if (overrides) {
-    Cli.options = overrides;
-  }
-
-  return await BuildParameters.create();
-}
-
-describe('Cloud Runner Custom Hooks And Steps', () => {
-  it('Responds', () => {});
-  setups();
-  it('Check parsing and reading of steps', async () => {
-    const yamlString = `hook: before
-commands: echo "test"`;
-    const yamlString2 = `- hook: before
-  commands: echo "test"`;
-    const overrides = {
-      versioning: 'None',
-      projectPath: 'test-project',
-      unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
-      targetPlatform: 'StandaloneLinux64',
-      cacheKey: `test-case-${uuidv4()}`,
-    };
-    CloudRunner.setup(await CreateParameters(overrides));
-    const stringObject = CloudRunnerCustomSteps.ParseSteps(yamlString);
-    const stringObject2 = CloudRunnerCustomSteps.ParseSteps(yamlString2);
-
-    CloudRunnerLogger.log(yamlString);
-    CloudRunnerLogger.log(JSON.stringify(stringObject, undefined, 4));
-
-    expect(stringObject.length).toBe(1);
-    expect(stringObject[0].hook).toBe(`before`);
-    expect(stringObject2.length).toBe(1);
-    expect(stringObject2[0].hook).toBe(`before`);
-
-    const getCustomStepsFromFiles = CloudRunnerCustomSteps.GetCustomStepsFromFiles(`before`);
-    CloudRunnerLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
-  });
-  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `k8s`) {
-    it('Run build once - check for pre and post custom hooks run contents', async () => {
-      const overrides = {
-        versioning: 'None',
-        projectPath: 'test-project',
-        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
-        targetPlatform: 'StandaloneLinux64',
-        cacheKey: `test-case-${uuidv4()}`,
-        customStepFiles: `my-test-step-pre-build,my-test-step-post-build`,
-      };
-      const buildParameter2 = await CreateParameters(overrides);
-      const baseImage2 = new ImageTag(buildParameter2);
-      const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
-      CloudRunnerLogger.log(`run 2 succeeded`);
-
-      const build2ContainsBuildSucceeded = results2.includes('Build succeeded');
-      const build2ContainsPreBuildHookRunMessage = results2.includes('before-build hook test!');
-      const build2ContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
-
-      const build2ContainsPreBuildStepMessage = results2.includes('before-build step test!');
-      const build2ContainsPostBuildStepMessage = results2.includes('after-build step test!');
-
-      expect(build2ContainsBuildSucceeded).toBeTruthy();
-      expect(build2ContainsPreBuildHookRunMessage).toBeTruthy();
-      expect(build2ContainsPostBuildHookRunMessage).toBeTruthy();
-      expect(build2ContainsPreBuildStepMessage).toBeTruthy();
-      expect(build2ContainsPostBuildStepMessage).toBeTruthy();
-    }, 1_000_000_000);
-  }
-});

src/model/cloud-runner/tests/cloud-runner-s3-steps.test.ts

@@ -2,11 +2,11 @@ import CloudRunner from '../cloud-runner';
 import { BuildParameters, ImageTag } from '../..';
 import UnityVersioning from '../../unity-versioning';
 import { Cli } from '../../cli/cli';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
 import { v4 as uuidv4 } from 'uuid';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from '../options/cloud-runner-options';
 import setups from './cloud-runner-suite.test';
-import { CloudRunnerSystem } from '../services/cloud-runner-system';
+import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
 import { OptionValues } from 'commander';
 
 async function CreateParameters(overrides: OptionValues | undefined) {
@@ -20,7 +20,7 @@ async function CreateParameters(overrides: OptionValues | undefined) {
 describe('Cloud Runner pre-built S3 steps', () => {
   it('Responds', () => {});
   setups();
-  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `local-docker`) {
+  if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
     it('Run build and prebuilt s3 cache pull, cache push and upload build', async () => {
       const overrides = {
         versioning: 'None',
@@ -28,7 +28,7 @@ describe('Cloud Runner pre-built S3 steps', () => {
         unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
         targetPlatform: 'StandaloneLinux64',
         cacheKey: `test-case-${uuidv4()}`,
-        customStepFiles: `aws-s3-pull-cache,aws-s3-upload-cache,aws-s3-upload-build`,
+        containerHookFiles: `aws-s3-pull-cache,aws-s3-upload-cache,aws-s3-upload-build`,
       };
       const buildParameter2 = await CreateParameters(overrides);
       const baseImage2 = new ImageTag(buildParameter2);
@@ -39,7 +39,7 @@ describe('Cloud Runner pre-built S3 steps', () => {
       expect(build2ContainsBuildSucceeded).toBeTruthy();
 
       const results = await CloudRunnerSystem.RunAndReadLines(
-        `aws s3 ls s3://${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/${buildParameter2.cacheKey}/`,
+        `aws s3 ls s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/`,
       );
       CloudRunnerLogger.log(results.join(`,`));
     }, 1_000_000_000);

src/model/cloud-runner/tests/create-test-parameter.ts

@@ -0,0 +1,8 @@
+import BuildParameters from '../../build-parameters';
+import { Cli } from '../../cli/cli';
+
+export async function CreateParameters(overrides: any) {
+  if (overrides) Cli.options = overrides;
+
+  return BuildParameters.create();
+}

src/model/cloud-runner/tests/e2e/cloud-runner-end2end-caching.test.ts

@@ -1,15 +1,15 @@
-import CloudRunner from '../cloud-runner';
+import CloudRunner from '../../cloud-runner';
-import { BuildParameters, ImageTag } from '../..';
+import { BuildParameters, ImageTag } from '../../..';
-import UnityVersioning from '../../unity-versioning';
+import UnityVersioning from '../../../unity-versioning';
-import { Cli } from '../../cli/cli';
+import { Cli } from '../../../cli/cli';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
 import { v4 as uuidv4 } from 'uuid';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
-import setups from './cloud-runner-suite.test';
+import setups from '../cloud-runner-suite.test';
-import fs from 'node:fs';
+import * as fs from 'node:fs';
-import { OptionValues } from 'commander';
+import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
 
-async function CreateParameters(overrides: OptionValues | undefined) {
+async function CreateParameters(overrides: any) {
   if (overrides) {
     Cli.options = overrides;
   }
@@ -28,10 +28,10 @@ describe('Cloud Runner Caching', () => {
         unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
         targetPlatform: 'StandaloneLinux64',
         cacheKey: `test-case-${uuidv4()}`,
-        customStepFiles: `debug-cache`,
+        containerHookFiles: `debug-cache`,
       };
-      if (CloudRunnerOptions.cloudRunnerCluster === `k8s`) {
+      if (CloudRunnerOptions.providerStrategy === `k8s`) {
-        overrides.customStepFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
+        overrides.containerHookFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
       }
       const buildParameter = await CreateParameters(overrides);
       expect(buildParameter.projectPath).toEqual(overrides.projectPath);
@@ -48,7 +48,14 @@ describe('Cloud Runner Caching', () => {
 
       CloudRunnerLogger.log(`run 1 succeeded`);
 
-      if (CloudRunnerOptions.cloudRunnerCluster === `local-docker`) {
+      if (CloudRunnerOptions.providerStrategy === `local-docker`) {
+        await CloudRunnerSystem.Run(`tree ./cloud-runner-cache/cache`);
+        await CloudRunnerSystem.Run(
+          `cp ./cloud-runner-cache/cache/${buildParameter.cacheKey}/Library/lib-${buildParameter.buildGuid}.tar ./`,
+        );
+        await CloudRunnerSystem.Run(`mkdir results`);
+        await CloudRunnerSystem.Run(`tar -xf lib-${buildParameter.buildGuid}.tar -C ./results`);
+        await CloudRunnerSystem.Run(`tree -d ./results`);
         const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
         expect(cacheFolderExists).toBeTruthy();
       }

src/model/cloud-runner/tests/e2e/cloud-runner-end2end-locking.test.ts

@@ -0,0 +1,92 @@
+import CloudRunner from '../../cloud-runner';
+import { BuildParameters } from '../../..';
+import UnityVersioning from '../../../unity-versioning';
+import { Cli } from '../../../cli/cli';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
+import { v4 as uuidv4 } from 'uuid';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
+import setups from '../cloud-runner-suite.test';
+import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking';
+
+async function CreateParameters(overrides: any) {
+  if (overrides) {
+    Cli.options = overrides;
+  }
+
+  return await BuildParameters.create();
+}
+
+describe('Cloud Runner Locking', () => {
+  setups();
+  it('Responds', () => {});
+  if (CloudRunnerOptions.cloudRunnerDebug) {
+    it(`Simple Locking End2End Flow`, async () => {
+      const overrides: any = {
+        versioning: 'None',
+        projectPath: 'test-project',
+        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
+        targetPlatform: 'StandaloneLinux64',
+        cacheKey: `test-case-${uuidv4()}`,
+        maxRetainedWorkspaces: 3,
+      };
+      const buildParameters = await CreateParameters(overrides);
+
+      const newWorkspaceName = `test-workspace-${uuidv4()}`;
+      const runId = uuidv4();
+      CloudRunner.buildParameters = buildParameters;
+      await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
+      expect(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
+      const isExpectedUnlockedBeforeLocking =
+        (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
+      expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
+      const result = await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
+      expect(result).toBeTruthy();
+      const lines = await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`);
+      expect(lines.map((x) => x.replace(`/`, ``)).includes(buildParameters.cacheKey));
+      expect(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParameters)).toBeTruthy();
+      expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
+      const allLocks = await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters);
+      expect(
+        (
+          await SharedWorkspaceLocking.ReadLines(
+            `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParameters.cacheKey}/`,
+          )
+        ).filter((x) => x.endsWith(`${newWorkspaceName}_workspace_lock`)),
+      ).toHaveLength(1);
+      expect(
+        (
+          await SharedWorkspaceLocking.ReadLines(
+            `aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParameters.cacheKey}/`,
+          )
+        ).filter((x) => x.endsWith(`${newWorkspaceName}_workspace`)),
+      ).toHaveLength(1);
+      expect(allLocks.filter((x) => x.endsWith(`${newWorkspaceName}_workspace_lock`)).length).toBeGreaterThan(0);
+      const isExpectedLockedAfterLocking =
+        (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true;
+      expect(isExpectedLockedAfterLocking).toBeTruthy();
+      const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocksForWorkspace(
+        newWorkspaceName,
+        buildParameters,
+      );
+      CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
+      expect(locksBeforeRelease.length).toBe(1);
+      await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
+      const locks = await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters);
+      expect(locks.length).toBe(0);
+      const isExpectedNotLockedAfterReleasing =
+        (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
+      expect(isExpectedNotLockedAfterReleasing).toBeTruthy();
+      const lockingResult2 = await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
+      expect(lockingResult2).toBeTruthy();
+      expect((await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true).toBeTruthy();
+      await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
+      expect(
+        (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false,
+      ).toBeTruthy();
+      await SharedWorkspaceLocking.CleanupWorkspace(newWorkspaceName, buildParameters);
+      CloudRunnerLogger.log(`Starting get or create`);
+      expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
+    }, 350000);
+  }
+});

src/model/cloud-runner/tests/e2e/cloud-runner-end2end-retaining.test.ts

@@ -1,24 +1,16 @@
-import CloudRunner from '../cloud-runner';
+import CloudRunner from '../../cloud-runner';
-import { BuildParameters, ImageTag } from '../..';
+import { ImageTag } from '../../..';
-import UnityVersioning from '../../unity-versioning';
+import UnityVersioning from '../../../unity-versioning';
-import { Cli } from '../../cli/cli';
+import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
 import { v4 as uuidv4 } from 'uuid';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from '../../options/cloud-runner-options';
-import setups from './cloud-runner-suite.test';
+import setups from './../cloud-runner-suite.test';
-import fs from 'node:fs';
+import * as fs from 'node:fs';
 import path from 'node:path';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
-import SharedWorkspaceLocking from '../services/shared-workspace-locking';
+import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking';
-import { OptionValues } from 'commander';
+import { CreateParameters } from '../create-test-parameter';
-
+import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
-async function CreateParameters(overrides: OptionValues | undefined) {
-  if (overrides) {
-    Cli.options = overrides;
-  }
-
-  return await BuildParameters.create();
-}
 
 describe('Cloud Runner Retain Workspace', () => {
   it('Responds', () => {});
@@ -31,7 +23,7 @@ describe('Cloud Runner Retain Workspace', () => {
         unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
         targetPlatform: 'StandaloneLinux64',
         cacheKey: `test-case-${uuidv4()}`,
-        retainWorkspaces: true,
+        maxRetainedWorkspaces: 1,
       };
       const buildParameter = await CreateParameters(overrides);
       expect(buildParameter.projectPath).toEqual(overrides.projectPath);
@@ -46,12 +38,15 @@ describe('Cloud Runner Retain Workspace', () => {
       expect(results).toContain(buildSucceededString);
       expect(results).not.toContain(cachePushFail);
 
-      if (CloudRunnerOptions.cloudRunnerCluster === `local-docker`) {
+      if (CloudRunnerOptions.providerStrategy === `local-docker`) {
         const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
         expect(cacheFolderExists).toBeTruthy();
+        await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache`);
       }
 
       CloudRunnerLogger.log(`run 1 succeeded`);
+
+      // await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache/${}`);
       const buildParameter2 = await CreateParameters(overrides);
 
       buildParameter2.cacheKey = buildParameter.cacheKey;

src/model/cloud-runner/tests/shared-workspace-locking.test.ts

@@ -1,102 +0,0 @@
-import SharedWorkspaceLocking from '../services/shared-workspace-locking';
-import { Cli } from '../../cli/cli';
-import setups from './cloud-runner-suite.test';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
-import { v4 as uuidv4 } from 'uuid';
-import CloudRunnerOptions from '../cloud-runner-options';
-import UnityVersioning from '../../unity-versioning';
-import BuildParameters from '../../build-parameters';
-import CloudRunner from '../cloud-runner';
-import { OptionValues } from 'commander';
-
-async function CreateParameters(overrides: OptionValues | undefined) {
-  if (overrides) {
-    Cli.options = overrides;
-  }
-
-  return await BuildParameters.create();
-}
-
-describe('Cloud Runner Locking', () => {
-  setups();
-  it('Responds', () => {});
-  if (CloudRunnerOptions.cloudRunnerDebug) {
-    it(`Simple Locking Flow`, async () => {
-      Cli.options!.retainWorkspaces = true;
-      const overrides: any = {
-        versioning: 'None',
-        projectPath: 'test-project',
-        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
-        targetPlatform: 'StandaloneLinux64',
-        cacheKey: `test-case-${uuidv4()}`,
-      };
-      const buildParameters = await CreateParameters(overrides);
-
-      const newWorkspaceName = `test-workspace-${uuidv4()}`;
-      const runId = uuidv4();
-      CloudRunner.buildParameters = buildParameters;
-      await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
-      const isExpectedUnlockedBeforeLocking =
-        (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
-      expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
-      await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
-      const isExpectedLockedAfterLocking =
-        (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true;
-      expect(isExpectedLockedAfterLocking).toBeTruthy();
-      const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
-      CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
-      expect(locksBeforeRelease.length).toBe(1);
-      await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
-      const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
-      expect(locks.length).toBe(0);
-      const isExpectedLockedAfterReleasing =
-        (await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
-      expect(isExpectedLockedAfterReleasing).toBeTruthy();
-    }, 150000);
-    it.skip('All Locking Actions', async () => {
-      Cli.options!.retainWorkspaces = true;
-      const overrides: OptionValues = {
-        versioning: 'None',
-        projectPath: 'test-project',
-        unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
-        targetPlatform: 'StandaloneLinux64',
-        cacheKey: `test-case-${uuidv4()}`,
-      };
-      const buildParameters = await CreateParameters(overrides);
-
-      CloudRunnerLogger.log(
-        `GetAllWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters))}`,
-      );
-      CloudRunnerLogger.log(
-        `GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
-      );
-      CloudRunnerLogger.log(
-        `IsWorkspaceLocked ${JSON.stringify(
-          await SharedWorkspaceLocking.IsWorkspaceLocked(`test-workspace-${uuidv4()}`, buildParameters),
-        )}`,
-      );
-      CloudRunnerLogger.log(
-        `GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
-      );
-      CloudRunnerLogger.log(
-        `LockWorkspace ${JSON.stringify(
-          await SharedWorkspaceLocking.LockWorkspace(`test-workspace-${uuidv4()}`, uuidv4(), buildParameters),
-        )}`,
-      );
-      CloudRunnerLogger.log(
-        `CreateLockableWorkspace ${JSON.stringify(
-          await SharedWorkspaceLocking.CreateWorkspace(`test-workspace-${uuidv4()}`, buildParameters),
-        )}`,
-      );
-      CloudRunnerLogger.log(
-        `GetLockedWorkspace ${JSON.stringify(
-          await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
-            `test-workspace-${uuidv4()}`,
-            uuidv4(),
-            buildParameters,
-          ),
-        )}`,
-      );
-    }, 3000000);
-  }
-});

src/model/cloud-runner/workflows/async-workflow.ts

@@ -1,7 +1,7 @@
-import CloudRunnerSecret from '../services/cloud-runner-secret';
+import CloudRunnerSecret from '../options/cloud-runner-secret';
-import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
 import CloudRunner from '../cloud-runner';
 
 export class AsyncWorkflow {
@@ -11,6 +11,9 @@ export class AsyncWorkflow {
   ): Promise<string> {
     try {
       CloudRunnerLogger.log(`Cloud Runner is running async mode`);
+      const asyncEnvironmentVariable = new CloudRunnerEnvironmentVariable();
+      asyncEnvironmentVariable.name = `ASYNC_WORKFLOW`;
+      asyncEnvironmentVariable.value = `true`;
 
       let output = '';
 
@@ -34,7 +37,7 @@ aws --version
 node /builder/dist/index.js -m async-workflow`,
         `/${CloudRunnerFolders.buildVolumeFolder}`,
         `/${CloudRunnerFolders.buildVolumeFolder}/`,
-        environmentVariables,
+        [...environmentVariables, asyncEnvironmentVariable],
         [
           ...secrets,
           ...[

src/model/cloud-runner/workflows/build-automation-workflow.ts

@@ -1,26 +1,25 @@
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
-import { CloudRunnerStepState } from '../cloud-runner-step-state';
+import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
 import { WorkflowInterface } from './workflow-interface';
 import * as core from '@actions/core';
-import { CloudRunnerCustomHooks } from '../services/cloud-runner-custom-hooks';
+import { CommandHookService } from '../services/hooks/command-hook-service';
 import path from 'node:path';
 import CloudRunner from '../cloud-runner';
-import CloudRunnerOptions from '../cloud-runner-options';
+import { ContainerHookService } from '../services/hooks/container-hook-service';
-import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
 
 export class BuildAutomationWorkflow implements WorkflowInterface {
-  async run(cloudRunnerStepState: CloudRunnerStepState) {
+  async run(cloudRunnerStepState: CloudRunnerStepParameters) {
     return await BuildAutomationWorkflow.standardBuildAutomation(cloudRunnerStepState.image, cloudRunnerStepState);
   }
 
-  private static async standardBuildAutomation(baseImage: string, cloudRunnerStepState: CloudRunnerStepState) {
+  private static async standardBuildAutomation(baseImage: string, cloudRunnerStepState: CloudRunnerStepParameters) {
     // TODO accept post and pre build steps as yaml files in the repo
     CloudRunnerLogger.log(`Cloud Runner is running standard build automation`);
 
     let output = '';
 
-    output += await CloudRunnerCustomSteps.RunPreBuildSteps(cloudRunnerStepState);
+    output += await ContainerHookService.RunPreBuildSteps(cloudRunnerStepState);
     CloudRunnerLogger.logWithTime('Configurable pre build step(s) time');
 
     if (!CloudRunner.buildParameters.isCliMode) core.startGroup('build');
@@ -40,7 +39,7 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
     if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
     CloudRunnerLogger.logWithTime('Build time');
 
-    output += await CloudRunnerCustomSteps.RunPostBuildSteps(cloudRunnerStepState);
+    output += await ContainerHookService.RunPostBuildSteps(cloudRunnerStepState);
     CloudRunnerLogger.logWithTime('Configurable post build step(s) time');
 
     CloudRunnerLogger.log(`Cloud Runner finished running standard build automation`);
@@ -49,11 +48,11 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
   }
 
   private static get BuildWorkflow() {
-    const setupHooks = CloudRunnerCustomHooks.getHooks(CloudRunner.buildParameters.customJobHooks).filter((x) =>
+    const setupHooks = CommandHookService.getHooks(CloudRunner.buildParameters.commandHooks).filter((x) =>
-      x.step.includes(`setup`),
+      x.step?.includes(`setup`),
     );
-    const buildHooks = CloudRunnerCustomHooks.getHooks(CloudRunner.buildParameters.customJobHooks).filter((x) =>
+    const buildHooks = CommandHookService.getHooks(CloudRunner.buildParameters.commandHooks).filter((x) =>
-      x.step.includes(`build`),
+      x.step?.includes(`build`),
     );
     const builderPath = CloudRunnerFolders.ToLinuxFolder(
       path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`),
@@ -65,16 +64,14 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
       n 16.15.1 > /dev/null
       npm --version
       node --version
-      ${BuildAutomationWorkflow.TreeCommand}
       ${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
       export GITHUB_WORKSPACE="${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}"
+      df -H /data/
       ${BuildAutomationWorkflow.setupCommands(builderPath)}
       ${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
-      ${BuildAutomationWorkflow.TreeCommand}
       ${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
       ${BuildAutomationWorkflow.BuildCommands(builderPath)}
-      ${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
+      ${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`;
-      ${BuildAutomationWorkflow.TreeCommand}`;
   }
 
   private static setupCommands(builderPath: string) {
@@ -84,24 +81,17 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
       CloudRunnerFolders.unityBuilderRepoUrl
     } "${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)}" && chmod +x ${builderPath}`;
 
-    const retainedWorkspaceCommands = `if [ -e "${CloudRunnerFolders.ToLinuxFolder(
-      CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
-    )}" ] && [ -e "${CloudRunnerFolders.ToLinuxFolder(
-      path.join(CloudRunnerFolders.repoPathAbsolute, `.git`),
-    )}" ]; then echo "Retained Workspace Already Exists!" ; fi`;
-
     const cloneBuilderCommands = `if [ -e "${CloudRunnerFolders.ToLinuxFolder(
       CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
     )}" ] && [ -e "${CloudRunnerFolders.ToLinuxFolder(
       path.join(CloudRunnerFolders.builderPathAbsolute, `.git`),
-    )}" ]; then echo "Builder Already Exists!"; else ${commands}; fi`;
+    )}" ] ; then echo "Builder Already Exists!" && tree ${
+      CloudRunnerFolders.builderPathAbsolute
+    }; else ${commands} ; fi`;
 
     return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
-    echo "downloading game-ci..."
+${cloneBuilderCommands}
-    ${retainedWorkspaceCommands}
+node ${builderPath} -m remote-cli-pre-build`;
-    ${cloneBuilderCommands}
-    echo "bootstrap game ci cloud runner..."
-    node ${builderPath} -m remote-cli-pre-build`;
   }
 
   private static BuildCommands(builderPath: string) {
@@ -122,10 +112,4 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
     chmod +x ${builderPath}
     node ${builderPath} -m remote-cli-post-build`;
   }
-
-  private static get TreeCommand(): string {
-    return CloudRunnerOptions.cloudRunnerDebugTree
-      ? `tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${CloudRunnerFolders.cacheFolderForCacheKeyFull} && du -h -s /${CloudRunnerFolders.buildVolumeFolder}/ && du -h -s ${CloudRunnerFolders.cacheFolderForAllFull}`
-      : `#`;
-  }
 }

src/model/cloud-runner/workflows/custom-workflow.ts

@@ -1,37 +1,37 @@
-import CloudRunnerLogger from '../services/cloud-runner-logger';
+import CloudRunnerLogger from '../services/core/cloud-runner-logger';
-import CloudRunnerSecret from '../services/cloud-runner-secret';
+import CloudRunnerSecret from '../options/cloud-runner-secret';
-import { CloudRunnerFolders } from '../services/cloud-runner-folders';
+import { CloudRunnerFolders } from '../options/cloud-runner-folders';
-import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
+import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
-import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
+import { ContainerHookService } from '../services/hooks/container-hook-service';
-import { CustomStep } from '../services/custom-step';
+import { ContainerHook } from '../services/hooks/container-hook';
 import CloudRunner from '../cloud-runner';
 
 export class CustomWorkflow {
-  public static async runCustomJobFromString(
+  public static async runContainerJobFromString(
     buildSteps: string,
     environmentVariables: CloudRunnerEnvironmentVariable[],
     secrets: CloudRunnerSecret[],
   ): Promise<string> {
-    return await CustomWorkflow.runCustomJob(
+    return await CustomWorkflow.runContainerJob(
-      CloudRunnerCustomSteps.ParseSteps(buildSteps),
+      ContainerHookService.ParseContainerHooks(buildSteps),
       environmentVariables,
       secrets,
     );
   }
 
-  public static async runCustomJob(
+  public static async runContainerJob(
-    buildSteps: CustomStep[],
+    steps: ContainerHook[],
     environmentVariables: CloudRunnerEnvironmentVariable[],
     secrets: CloudRunnerSecret[],
   ) {
     try {
-      CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
       let output = '';
 
       // if (CloudRunner.buildParameters?.cloudRunnerDebug) {
       //   CloudRunnerLogger.log(`Custom Job Description \n${JSON.stringify(buildSteps, undefined, 4)}`);
       // }
-      for (const step of buildSteps) {
+      for (const step of steps) {
+        CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
         output += await CloudRunner.Provider.runTaskInWorkflow(
           CloudRunner.buildParameters.buildGuid,
           step.image,

src/model/cloud-runner/workflows/workflow-composition-root.ts

@@ -1,20 +1,24 @@
-import { CloudRunnerStepState } from '../cloud-runner-step-state';
+import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
 import { CustomWorkflow } from './custom-workflow';
 import { WorkflowInterface } from './workflow-interface';
 import { BuildAutomationWorkflow } from './build-automation-workflow';
 import CloudRunner from '../cloud-runner';
-import CloudRunnerOptions from '../cloud-runner-options';
+import CloudRunnerOptions from '../options/cloud-runner-options';
 import { AsyncWorkflow } from './async-workflow';
 
 export class WorkflowCompositionRoot implements WorkflowInterface {
-  async run(cloudRunnerStepState: CloudRunnerStepState) {
+  async run(cloudRunnerStepState: CloudRunnerStepParameters) {
     try {
-      if (CloudRunnerOptions.asyncCloudRunner) {
+      if (
+        CloudRunnerOptions.asyncCloudRunner &&
+        !CloudRunner.isCloudRunnerAsyncEnvironment &&
+        !CloudRunner.isCloudRunnerEnvironment
+      ) {
         return await AsyncWorkflow.runAsyncWorkflow(cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
       }
 
       if (CloudRunner.buildParameters.customJob !== '') {
-        return await CustomWorkflow.runCustomJobFromString(
+        return await CustomWorkflow.runContainerJobFromString(
           CloudRunner.buildParameters.customJob,
           cloudRunnerStepState.environment,
           cloudRunnerStepState.secrets,
@@ -22,7 +26,7 @@ export class WorkflowCompositionRoot implements WorkflowInterface {
       }
 
       return await new BuildAutomationWorkflow().run(
-        new CloudRunnerStepState(
+        new CloudRunnerStepParameters(
           cloudRunnerStepState.image.toString(),
           cloudRunnerStepState.environment,
           cloudRunnerStepState.secrets,

src/model/cloud-runner/workflows/workflow-interface.ts

@@ -1,8 +1,8 @@
-import { CloudRunnerStepState } from '../cloud-runner-step-state';
+import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
 
 export interface WorkflowInterface {
   run(
     // eslint-disable-next-line no-unused-vars
-    cloudRunnerStepState: CloudRunnerStepState,
+    cloudRunnerStepState: CloudRunnerStepParameters,
   ): Promise<string>;
 }

src/model/docker.ts

@@ -15,6 +15,7 @@ class Docker {
     // eslint-disable-next-line unicorn/no-useless-undefined
     options: ExecOptions | undefined = undefined,
     entrypointBash: boolean = false,
+    errorWhenMissingUnityBuildResults: boolean = true,
   ) {
     let runCommand = '';
     switch (process.platform) {
@@ -26,9 +27,9 @@ class Docker {
     }
     if (options) {
       options.silent = silent;
-      await execWithErrorCheck(runCommand, undefined, options);
+      await execWithErrorCheck(runCommand, undefined, options, errorWhenMissingUnityBuildResults);
     } else {
-      await execWithErrorCheck(runCommand, undefined, { silent });
+      await execWithErrorCheck(runCommand, undefined, { silent }, errorWhenMissingUnityBuildResults);
     }
   }
 

src/model/exec-with-error-check.ts

@@ -4,9 +4,14 @@ export async function execWithErrorCheck(
   commandLine: string,
   arguments_?: string[],
   options?: ExecOptions,
+  errorWhenMissingUnityBuildResults: boolean = true,
 ): Promise<number> {
   const result = await getExecOutput(commandLine, arguments_, options);
 
+  if (!errorWhenMissingUnityBuildResults) {
+    return result.exitCode;
+  }
+
   // Check for errors in the Build Results section
   const match = result.stdout.match(/^#\s*Build results\s*#(.*)^Size:/ms);
 

src/model/github.ts

@@ -1,6 +1,6 @@
-import CloudRunnerLogger from './cloud-runner/services/cloud-runner-logger';
+import CloudRunnerLogger from './cloud-runner/services/core/cloud-runner-logger';
 import CloudRunner from './cloud-runner/cloud-runner';
-import CloudRunnerOptions from './cloud-runner/cloud-runner-options';
+import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
 import * as core from '@actions/core';
 import { Octokit } from '@octokit/core';
 
@@ -10,6 +10,7 @@ class GitHub {
   private static longDescriptionContent: string = ``;
   private static startedDate: string;
   private static endedDate: string;
+  static result: string = ``;
   private static get octokitDefaultToken() {
     return new Octokit({
       auth: process.env.GITHUB_TOKEN,
@@ -33,7 +34,7 @@ class GitHub {
   }
 
   private static get checkRunId() {
-    return CloudRunner.githubCheckId;
+    return CloudRunner.buildParameters.githubCheckId;
   }
 
   private static get owner() {
@@ -45,13 +46,12 @@ class GitHub {
   }
 
   public static async createGitHubCheck(summary: string) {
-    if (!CloudRunnerOptions.githubChecks) {
+    if (!CloudRunner.buildParameters.githubChecks) {
       return ``;
     }
     GitHub.startedDate = new Date().toISOString();
 
-    CloudRunnerLogger.log(`POST /repos/${GitHub.owner}/${GitHub.repo}/check-runs`);
+    CloudRunnerLogger.log(`Creating inital github check`);
-
     const data = {
       owner: GitHub.owner,
       repo: GitHub.repo,
@@ -78,20 +78,27 @@ class GitHub {
     };
     const result = await GitHub.createGitHubCheckRequest(data);
 
-    return result.data.id;
+    return result.data.id.toString();
   }
 
   public static async updateGitHubCheck(
     longDescription: string,
-    summary: any,
+    summary: string,
     result = `neutral`,
     status = `in_progress`,
   ) {
-    if (!CloudRunnerOptions.githubChecks) {
+    if (`${CloudRunner.buildParameters.githubChecks}` !== `true`) {
       return;
     }
+    CloudRunnerLogger.log(
+      `githubChecks: ${CloudRunner.buildParameters.githubChecks} checkRunId: ${GitHub.checkRunId} sha: ${GitHub.sha} async: ${CloudRunner.isCloudRunnerAsyncEnvironment}`,
+    );
     GitHub.longDescriptionContent += `\n${longDescription}`;
-
+    if (GitHub.result !== `success` && GitHub.result !== `failure`) {
+      GitHub.result = result;
+    } else {
+      result = GitHub.result;
+    }
     const data: any = {
       owner: GitHub.owner,
       repo: GitHub.repo,
@@ -120,12 +127,9 @@ class GitHub {
       data.conclusion = result;
     }
 
-    if (await CloudRunnerOptions.asyncCloudRunner) {
+    await (CloudRunner.isCloudRunnerAsyncEnvironment
-      await GitHub.runUpdateAsyncChecksWorkflow(data, `update`);
+      ? GitHub.runUpdateAsyncChecksWorkflow(data, `update`)
-
+      : GitHub.updateGitHubCheckRequest(data));
-      return;
-    }
-    await GitHub.updateGitHubCheckRequest(data);
   }
 
   public static async updateGitHubCheckRequest(data: any) {
@@ -140,18 +144,16 @@ class GitHub {
     if (mode === `create`) {
       throw new Error(`Not supported: only use update`);
     }
-    const workflowsResult = await GitHub.octokitDefaultToken.request(
+    const workflowsResult = await GitHub.octokitPAT.request(`GET /repos/{owner}/{repo}/actions/workflows`, {
-      `GET /repos/${GitHub.owner}/${GitHub.repo}/actions/workflows`,
+      owner: GitHub.owner,
-      {
+      repo: GitHub.repo,
-        owner: GitHub.owner,
+    });
-        repo: GitHub.repo,
-      },
-    );
     const workflows = workflowsResult.data.workflows;
+    CloudRunnerLogger.log(`Got ${workflows.length} workflows`);
     let selectedId = ``;
     for (let index = 0; index < workflowsResult.data.total_count; index++) {
       if (workflows[index].name === GitHub.asyncChecksApiWorkflowName) {
-        selectedId = workflows[index].id;
+        selectedId = workflows[index].id.toString();
       }
     }
     if (selectedId === ``) {
@@ -169,6 +171,41 @@ class GitHub {
       },
     });
   }
+
+  static async triggerWorkflowOnComplete(triggerWorkflowOnComplete: string[]) {
+    const isLocalAsync = CloudRunner.buildParameters.asyncWorkflow && !CloudRunner.isCloudRunnerAsyncEnvironment;
+    if (isLocalAsync) {
+      return;
+    }
+    const workflowsResult = await GitHub.octokitPAT.request(`GET /repos/{owner}/{repo}/actions/workflows`, {
+      owner: GitHub.owner,
+      repo: GitHub.repo,
+    });
+    const workflows = workflowsResult.data.workflows;
+    CloudRunnerLogger.log(`Got ${workflows.length} workflows`);
+    for (const element of triggerWorkflowOnComplete) {
+      let selectedId = ``;
+      for (let index = 0; index < workflowsResult.data.total_count; index++) {
+        if (workflows[index].name === element) {
+          selectedId = workflows[index].id.toString();
+        }
+      }
+      if (selectedId === ``) {
+        core.info(JSON.stringify(workflows));
+        throw new Error(`no workflow with name "${GitHub.asyncChecksApiWorkflowName}"`);
+      }
+      await GitHub.octokitPAT.request(`POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches`, {
+        owner: GitHub.owner,
+        repo: GitHub.repo,
+        // eslint-disable-next-line camelcase
+        workflow_id: selectedId,
+        ref: CloudRunnerOptions.branch,
+        inputs: {
+          buildGuid: CloudRunner.buildParameters.buildGuid,
+        },
+      });
+    }
+  }
 }
 
 export default GitHub;

src/model/image-environment-factory.ts

@@ -65,7 +65,7 @@ class ImageEnvironmentFactory {
       { name: 'RUNNER_TEMP', value: process.env.RUNNER_TEMP },
       { name: 'RUNNER_WORKSPACE', value: process.env.RUNNER_WORKSPACE },
     ];
-    if (parameters.cloudRunnerCluster === 'local-docker') {
+    if (parameters.providerStrategy === 'local-docker') {
       for (const element of additionalVariables) {
         if (
           environmentVariables.find(

src/model/input-readers/generic-input-reader.ts

@@ -1,9 +1,9 @@
-import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
+import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
-import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
+import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
 
 export class GenericInputReader {
   public static async Run(command: string) {
-    if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
+    if (CloudRunnerOptions.providerStrategy === 'local') {
       return '';
     }
 

src/model/input-readers/git-repo.test.ts

@@ -1,6 +1,6 @@
 import { GitRepoReader } from './git-repo';
-import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
+import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
-import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
+import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
 
 describe(`git repo tests`, () => {
   it(`Branch value parsed from CLI to not contain illegal characters`, async () => {
@@ -11,14 +11,14 @@ describe(`git repo tests`, () => {
   it(`returns valid branch name when using https`, async () => {
     const mockValue = 'https://github.com/example/example.git';
     await jest.spyOn(CloudRunnerSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
-    await jest.spyOn(CloudRunnerOptions, 'cloudRunnerCluster', 'get').mockReturnValue('not-local');
+    await jest.spyOn(CloudRunnerOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
     expect(await GitRepoReader.GetRemote()).toEqual(`example/example`);
   });
 
   it(`returns valid branch name when using ssh`, async () => {
     const mockValue = 'git@github.com:example/example.git';
     await jest.spyOn(CloudRunnerSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
-    await jest.spyOn(CloudRunnerOptions, 'cloudRunnerCluster', 'get').mockReturnValue('not-local');
+    await jest.spyOn(CloudRunnerOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
     expect(await GitRepoReader.GetRemote()).toEqual(`example/example`);
   });
 });

src/model/input-readers/git-repo.ts

@@ -1,13 +1,13 @@
 import { assert } from 'node:console';
 import fs from 'node:fs';
-import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
+import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
-import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
+import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
-import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
+import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
 import Input from '../input';
 
 export class GitRepoReader {
   public static async GetRemote() {
-    if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
+    if (CloudRunnerOptions.providerStrategy === 'local') {
       return '';
     }
     assert(fs.existsSync(`.git`));
@@ -22,7 +22,7 @@ export class GitRepoReader {
   }
 
   public static async GetBranch() {
-    if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
+    if (CloudRunnerOptions.providerStrategy === 'local') {
       return '';
     }
     assert(fs.existsSync(`.git`));

src/model/input-readers/github-cli.ts

@@ -1,10 +1,10 @@
-import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
+import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
 import * as core from '@actions/core';
-import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
+import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
 
 export class GithubCliReader {
   static async GetGitHubAuthToken() {
-    if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
+    if (CloudRunnerOptions.providerStrategy === 'local') {
       return '';
     }
     try {

src/model/input-readers/test-license-reader.ts

@@ -1,10 +1,10 @@
 import path from 'node:path';
 import fs from 'node:fs';
 import YAML from 'yaml';
-import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
+import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
 
 export function ReadLicense(): string {
-  if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
+  if (CloudRunnerOptions.providerStrategy === 'local') {
     return '';
   }
   const pipelineFile = path.join(__dirname, `.github`, `workflows`, `cloud-runner-k8s-pipeline.yml`);

src/model/input.ts

@@ -1,7 +1,7 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { Cli } from './cli/cli';
-import CloudRunnerQueryOverride from './cloud-runner/services/cloud-runner-query-override';
+import CloudRunnerQueryOverride from './cloud-runner/options/cloud-runner-query-override';
 import Platform from './platform';
 import GitHub from './github';
 

yarn.lock

@@ -5946,6 +5946,11 @@ uuid@^8.3.0, uuid@^8.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
 
+uuid@^9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"
+  integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==
+
 v8-compile-cache@^2.0.3:
   version "2.3.0"
   resolved "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz"