Fix "large package handling" to redirect to shared folder (#473)

src/model/cloud-runner/cloud-runner-options.ts

@@ -98,7 +98,7 @@ class CloudRunnerOptions {
   }
 
   static get cloudRunnerBranch() {
-    return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'cloud-runner-develop';
+    return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
   }
 
   static get cloudRunnerCluster() {
@@ -220,21 +220,19 @@ class CloudRunnerOptions {
   }
 
   static get watchCloudRunnerToEnd(): boolean {
-    const input = CloudRunnerOptions.getInput(`watchToEnd`);
+    return (CloudRunnerOptions.getInput(`watchToEnd`) || true) !== 'false';
-
-    return input !== 'false';
   }
 
   public static get useSharedLargePackages(): boolean {
-    return CloudRunnerOptions.getInput(`useSharedLargePackages`) || false;
+    return (CloudRunnerOptions.getInput(`useSharedLargePackages`) || 'false') !== 'false';
   }
 
   public static get useSharedBuilder(): boolean {
-    return CloudRunnerOptions.getInput(`useSharedBuilder`) || true;
+    return (CloudRunnerOptions.getInput(`useSharedBuilder`) || true) !== 'false';
   }
 
   public static get useLz4Compression(): boolean {
-    return CloudRunnerOptions.getInput(`useLz4Compression`) || true;
+    return (CloudRunnerOptions.getInput(`useLz4Compression`) || true) !== false;
   }
 
   // ### ### ###

src/model/cloud-runner/providers/aws/index.ts

@@ -9,7 +9,6 @@ import CloudRunnerLogger from '../../services/cloud-runner-logger';
 import { AWSJobStack as AwsJobStack } from './aws-job-stack';
 import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
 import { Input } from '../../..';
-import { TertiaryResourcesService } from './services/tertiary-resources-service';
 import { GarbageCollectionService } from './services/garbage-collection-service';
 import { ProviderResource } from '../provider-resource';
 import { ProviderWorkflow } from '../provider-workflow';
@@ -22,10 +21,9 @@ class AWSBuildEnvironment implements ProviderInterface {
     this.baseStackName = buildParameters.awsBaseStackName;
   }
   async listResources(): Promise<ProviderResource[]> {
-    await TaskService.awsListJobs();
+    await TaskService.getCloudFormationJobStacks();
-    await TertiaryResourcesService.awsListLogGroups();
+    await TaskService.getLogGroups();
-    await TaskService.awsListTasks();
+    await TaskService.getTasks();
-    await TaskService.awsListStacks();
 
     return [];
   }
@@ -37,7 +35,7 @@ class AWSBuildEnvironment implements ProviderInterface {
   }
 
   async listOtherResources(): Promise<string> {
-    await TertiaryResourcesService.awsListLogGroups();
+    await TaskService.getLogGroups();
 
     return '';
   }

src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts

@@ -2,7 +2,6 @@ import AWS from 'aws-sdk';
 import Input from '../../../../input';
 import CloudRunnerLogger from '../../../services/cloud-runner-logger';
 import { TaskService } from './task-service';
-import { TertiaryResourcesService } from './tertiary-resources-service';
 
 export class GarbageCollectionService {
   static isOlderThan1day(date: any) {
@@ -17,14 +16,17 @@ export class GarbageCollectionService {
     const ecs = new AWS.ECS();
     const cwl = new AWS.CloudWatchLogs();
     const taskDefinitionsInUse = new Array();
-    await TaskService.awsListTasks(async (taskElement, element) => {
+    const tasks = await TaskService.getTasks();
+    for (const task of tasks) {
+      const { taskElement, element } = task;
       taskDefinitionsInUse.push(taskElement.taskDefinitionArn);
       if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) {
         CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`);
         await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise();
       }
-    });
+    }
-    await TaskService.awsListStacks(async (element) => {
+    const jobStacks = await TaskService.getCloudFormationJobStacks();
+    for (const element of jobStacks) {
       if (
         (await CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources?.some(
           (x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId),
@@ -44,8 +46,9 @@ export class GarbageCollectionService {
         const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName };
         await CF.deleteStack(deleteStackInput).promise();
       }
-    });
+    }
-    await TertiaryResourcesService.awsListLogGroups(async (element) => {
+    const logGroups = await TaskService.getLogGroups();
+    for (const element of logGroups) {
       if (
         deleteResources &&
         (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt)))
@@ -53,6 +56,11 @@ export class GarbageCollectionService {
         CloudRunnerLogger.log(`Deleting ${element.logGroupName}`);
         await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise();
       }
-    });
+    }
+
+    const locks = await TaskService.getLocks();
+    for (const element of locks) {
+      CloudRunnerLogger.log(`Lock: ${element.Key}`);
+    }
   }
 }

src/model/cloud-runner/providers/aws/services/task-service.ts

@@ -3,6 +3,7 @@ import Input from '../../../../input';
 import CloudRunnerLogger from '../../../services/cloud-runner-logger';
 import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
 import AwsTaskRunner from '../aws-task-runner';
+import { ListObjectsRequest } from 'aws-sdk/clients/s3';
 
 export class TaskService {
   static async watch() {
@@ -15,16 +16,19 @@ export class TaskService {
 
     return output;
   }
-  public static async awsListStacks(perResultCallback: any = false) {
+  public static async getCloudFormationJobStacks() {
-    CloudRunnerLogger.log(`List Stacks`);
+    const result: any[] = [];
+    CloudRunnerLogger.log(``);
+    CloudRunnerLogger.log(`List Cloud Formation Stacks`);
     process.env.AWS_REGION = Input.region;
     const CF = new AWS.CloudFormation();
     const stacks =
       (await CF.listStacks().promise()).StackSummaries?.filter(
-        (_x) => _x.StackStatus !== 'DELETE_COMPLETE', // &&
+        (_x) =>
-        // _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''),
+          _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
       ) || [];
-    CloudRunnerLogger.log(`Stacks ${stacks.length}`);
+    CloudRunnerLogger.log(``);
+    CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`);
     for (const element of stacks) {
       const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
 
@@ -33,13 +37,14 @@ export class TaskService {
           ageDate.getHours() / 24,
         )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
       );
-      if (perResultCallback) await perResultCallback(element);
+      result.push(element);
     }
     const baseStacks =
       (await CF.listStacks().promise()).StackSummaries?.filter(
         (_x) =>
           _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription,
       ) || [];
+    CloudRunnerLogger.log(``);
     CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`);
     for (const element of baseStacks) {
       const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
@@ -49,18 +54,20 @@ export class TaskService {
           ageDate.getHours() / 24,
         )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
       );
-      if (perResultCallback) await perResultCallback(element);
+      result.push(element);
-    }
-    if (stacks === undefined) {
-      return;
     }
+    CloudRunnerLogger.log(``);
+
+    return result;
   }
-  public static async awsListTasks(perResultCallback: any = false) {
+  public static async getTasks() {
+    const result: any[] = [];
+    CloudRunnerLogger.log(``);
     CloudRunnerLogger.log(`List Tasks`);
     process.env.AWS_REGION = Input.region;
     const ecs = new AWS.ECS();
     const clusters = (await ecs.listClusters().promise()).clusterArns || [];
-    CloudRunnerLogger.log(`Clusters ${clusters.length}`);
+    CloudRunnerLogger.log(`Task Clusters ${clusters.length}`);
     for (const element of clusters) {
       const input: AWS.ECS.ListTasksRequest = {
         cluster: element,
@@ -71,6 +78,7 @@ export class TaskService {
         const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element };
         const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || [];
         if (describeList.length === 0) {
+          CloudRunnerLogger.log(`No Tasks`);
           continue;
         }
         CloudRunnerLogger.log(`Tasks ${describeList.length}`);
@@ -84,31 +92,13 @@ export class TaskService {
             CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`);
             continue;
           }
-          if (perResultCallback) await perResultCallback(taskElement, element);
+          result.push({ taskElement, element });
         }
       }
     }
-  }
+    CloudRunnerLogger.log(``);
-  public static async awsListJobs(perResultCallback: any = false) {
-    CloudRunnerLogger.log(`List Jobs`);
-    process.env.AWS_REGION = Input.region;
-    const CF = new AWS.CloudFormation();
-    const stacks =
-      (await CF.listStacks().promise()).StackSummaries?.filter(
-        (_x) =>
-          _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription,
-      ) || [];
-    CloudRunnerLogger.log(`Stacks ${stacks.length}`);
-    for (const element of stacks) {
-      const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime());
 
-      CloudRunnerLogger.log(
+    return result;
-        `Task Stack ${element.StackName} - Age D${Math.floor(
-          ageDate.getHours() / 24,
-        )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
-      );
-      if (perResultCallback) await perResultCallback(element);
-    }
   }
   public static async awsDescribeJob(job: string) {
     process.env.AWS_REGION = Input.region;
@@ -131,4 +121,47 @@ export class TaskService {
 
     return message;
   }
+  public static async getLogGroups() {
+    const result: any[] = [];
+    process.env.AWS_REGION = Input.region;
+    const ecs = new AWS.CloudWatchLogs();
+    let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
+      /* logGroupNamePrefix: 'game-ci' */
+    };
+    let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
+    const logGroups = logGroupsDescribe.logGroups || [];
+    while (logGroupsDescribe.nextToken) {
+      logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
+      logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
+      logGroups.push(...(logGroupsDescribe?.logGroups || []));
+    }
+
+    CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
+    for (const element of logGroups) {
+      if (element.creationTime === undefined) {
+        CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
+        continue;
+      }
+      const ageDate: Date = new Date(Date.now() - element.creationTime);
+
+      CloudRunnerLogger.log(
+        `Task Stack ${element.logGroupName} - Age D${Math.floor(
+          ageDate.getHours() / 24,
+        )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
+      );
+      result.push(element);
+    }
+
+    return result;
+  }
+  public static async getLocks() {
+    process.env.AWS_REGION = Input.region;
+    const s3 = new AWS.S3();
+    const listRequest: ListObjectsRequest = {
+      Bucket: `game-ci-test-storage`,
+    };
+    const results = await s3.listObjects(listRequest).promise();
+
+    return results.Contents || [];
+  }
 }

src/model/cloud-runner/providers/aws/services/tertiary-resources-service.ts

@@ -1,36 +0,0 @@
-import AWS from 'aws-sdk';
-import Input from '../../../../input';
-import CloudRunnerLogger from '../../../services/cloud-runner-logger';
-
-export class TertiaryResourcesService {
-  public static async awsListLogGroups(perResultCallback: any = false) {
-    process.env.AWS_REGION = Input.region;
-    const ecs = new AWS.CloudWatchLogs();
-    let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = {
-      /* logGroupNamePrefix: 'game-ci' */
-    };
-    let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
-    const logGroups = logGroupsDescribe.logGroups || [];
-    while (logGroupsDescribe.nextToken) {
-      logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken };
-      logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise();
-      logGroups.push(...(logGroupsDescribe?.logGroups || []));
-    }
-
-    CloudRunnerLogger.log(`Log Groups ${logGroups.length}`);
-    for (const element of logGroups) {
-      if (element.creationTime === undefined) {
-        CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`);
-        continue;
-      }
-      const ageDate: Date = new Date(Date.now() - element.creationTime);
-
-      CloudRunnerLogger.log(
-        `Task Stack ${element.logGroupName} - Age D${Math.floor(
-          ageDate.getHours() / 24,
-        )} H${ageDate.getHours()} M${ageDate.getMinutes()}`,
-      );
-      if (perResultCallback) await perResultCallback(element, element);
-    }
-  }
-}

src/model/cloud-runner/remote-client/index.ts

@@ -105,15 +105,15 @@ export class RemoteClient {
   }
 
   static replaceLargePackageReferencesWithSharedReferences() {
-    const manifest = fs.readFileSync(
-      path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`),
-      'utf8',
-    );
-    if (CloudRunner.buildParameters.cloudRunnerDebug) {
-      CloudRunnerLogger.log(manifest);
-    }
     if (CloudRunner.buildParameters.useSharedLargePackages) {
-      manifest.replace(/LargePackages/g, '../../LargePackages');
+      const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
+      let manifest = fs.readFileSync(filePath, 'utf8');
+      manifest = manifest.replace(/LargeContent/g, '../../LargeContent');
+      fs.writeFileSync(filePath, manifest);
+      if (CloudRunner.buildParameters.cloudRunnerDebug) {
+        CloudRunnerLogger.log(`Package Manifest`);
+        CloudRunnerLogger.log(manifest);
+      }
     }
   }