diff --git a/dist/index.js b/dist/index.js index 3164940e..2689df0b 100644 Binary files a/dist/index.js and b/dist/index.js differ diff --git a/dist/index.js.map b/dist/index.js.map index ce190eee..a64d613e 100644 Binary files a/dist/index.js.map and b/dist/index.js.map differ diff --git a/src/model/cloud-runner/cloud-runner-options.ts b/src/model/cloud-runner/cloud-runner-options.ts index b8e3fa24..511c396c 100644 --- a/src/model/cloud-runner/cloud-runner-options.ts +++ b/src/model/cloud-runner/cloud-runner-options.ts @@ -98,7 +98,7 @@ class CloudRunnerOptions { } static get cloudRunnerBranch() { - return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'cloud-runner-develop'; + return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main'; } static get cloudRunnerCluster() { @@ -220,21 +220,19 @@ class CloudRunnerOptions { } static get watchCloudRunnerToEnd(): boolean { - const input = CloudRunnerOptions.getInput(`watchToEnd`); - - return input !== 'false'; + return (CloudRunnerOptions.getInput(`watchToEnd`) || true) !== 'false'; } public static get useSharedLargePackages(): boolean { - return CloudRunnerOptions.getInput(`useSharedLargePackages`) || false; + return (CloudRunnerOptions.getInput(`useSharedLargePackages`) || 'false') !== 'false'; } public static get useSharedBuilder(): boolean { - return CloudRunnerOptions.getInput(`useSharedBuilder`) || true; + return (CloudRunnerOptions.getInput(`useSharedBuilder`) || true) !== 'false'; } public static get useLz4Compression(): boolean { - return CloudRunnerOptions.getInput(`useLz4Compression`) || true; + return (CloudRunnerOptions.getInput(`useLz4Compression`) || true) !== false; } // ### ### ### diff --git a/src/model/cloud-runner/providers/aws/index.ts b/src/model/cloud-runner/providers/aws/index.ts index fa40e747..47b0faee 100644 --- a/src/model/cloud-runner/providers/aws/index.ts +++ b/src/model/cloud-runner/providers/aws/index.ts @@ -9,7 +9,6 @@ import CloudRunnerLogger from '../../services/cloud-runner-logger'; import { AWSJobStack as AwsJobStack } from './aws-job-stack'; import { AWSBaseStack as AwsBaseStack } from './aws-base-stack'; import { Input } from '../../..'; -import { TertiaryResourcesService } from './services/tertiary-resources-service'; import { GarbageCollectionService } from './services/garbage-collection-service'; import { ProviderResource } from '../provider-resource'; import { ProviderWorkflow } from '../provider-workflow'; @@ -22,10 +21,9 @@ class AWSBuildEnvironment implements ProviderInterface { this.baseStackName = buildParameters.awsBaseStackName; } async listResources(): Promise { - await TaskService.awsListJobs(); - await TertiaryResourcesService.awsListLogGroups(); - await TaskService.awsListTasks(); - await TaskService.awsListStacks(); + await TaskService.getCloudFormationJobStacks(); + await TaskService.getLogGroups(); + await TaskService.getTasks(); return []; } @@ -37,7 +35,7 @@ class AWSBuildEnvironment implements ProviderInterface { } async listOtherResources(): Promise { - await TertiaryResourcesService.awsListLogGroups(); + await TaskService.getLogGroups(); return ''; } diff --git a/src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts b/src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts index 51f58090..b46e660b 100644 --- a/src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts +++ b/src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts @@ -2,7 +2,6 @@ import AWS from 'aws-sdk'; import Input from '../../../../input'; import CloudRunnerLogger from '../../../services/cloud-runner-logger'; import { TaskService } from './task-service'; -import { TertiaryResourcesService } from './tertiary-resources-service'; export class GarbageCollectionService { static isOlderThan1day(date: any) { @@ -17,14 +16,17 @@ export class GarbageCollectionService { const ecs = new AWS.ECS(); const cwl = new AWS.CloudWatchLogs(); const taskDefinitionsInUse = new Array(); - await TaskService.awsListTasks(async (taskElement, element) => { + const tasks = await TaskService.getTasks(); + for (const task of tasks) { + const { taskElement, element } = task; taskDefinitionsInUse.push(taskElement.taskDefinitionArn); if (deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(taskElement.CreatedAt))) { CloudRunnerLogger.log(`Stopping task ${taskElement.containers?.[0].name}`); await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise(); } - }); - await TaskService.awsListStacks(async (element) => { + } + const jobStacks = await TaskService.getCloudFormationJobStacks(); + for (const element of jobStacks) { if ( (await CF.describeStackResources({ StackName: element.StackName }).promise()).StackResources?.some( (x) => x.ResourceType === 'AWS::ECS::TaskDefinition' && taskDefinitionsInUse.includes(x.PhysicalResourceId), @@ -44,8 +46,9 @@ export class GarbageCollectionService { const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName }; await CF.deleteStack(deleteStackInput).promise(); } - }); - await TertiaryResourcesService.awsListLogGroups(async (element) => { + } + const logGroups = await TaskService.getLogGroups(); + for (const element of logGroups) { if ( deleteResources && (!OneDayOlderOnly || GarbageCollectionService.isOlderThan1day(new Date(element.createdAt))) @@ -53,6 +56,11 @@ export class GarbageCollectionService { CloudRunnerLogger.log(`Deleting ${element.logGroupName}`); await cwl.deleteLogGroup({ logGroupName: element.logGroupName || '' }).promise(); } - }); + } + + const locks = await TaskService.getLocks(); + for (const element of locks) { + CloudRunnerLogger.log(`Lock: ${element.Key}`); + } } } diff --git a/src/model/cloud-runner/providers/aws/services/task-service.ts b/src/model/cloud-runner/providers/aws/services/task-service.ts index d8ba36f2..e9972854 100644 --- a/src/model/cloud-runner/providers/aws/services/task-service.ts +++ b/src/model/cloud-runner/providers/aws/services/task-service.ts @@ -3,6 +3,7 @@ import Input from '../../../../input'; import CloudRunnerLogger from '../../../services/cloud-runner-logger'; import { BaseStackFormation } from '../cloud-formations/base-stack-formation'; import AwsTaskRunner from '../aws-task-runner'; +import { ListObjectsRequest } from 'aws-sdk/clients/s3'; export class TaskService { static async watch() { @@ -15,16 +16,19 @@ export class TaskService { return output; } - public static async awsListStacks(perResultCallback: any = false) { - CloudRunnerLogger.log(`List Stacks`); + public static async getCloudFormationJobStacks() { + const result: any[] = []; + CloudRunnerLogger.log(``); + CloudRunnerLogger.log(`List Cloud Formation Stacks`); process.env.AWS_REGION = Input.region; const CF = new AWS.CloudFormation(); const stacks = (await CF.listStacks().promise()).StackSummaries?.filter( - (_x) => _x.StackStatus !== 'DELETE_COMPLETE', // && - // _x.TemplateDescription === TaskDefinitionFormation.description.replace('\n', ''), + (_x) => + _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription, ) || []; - CloudRunnerLogger.log(`Stacks ${stacks.length}`); + CloudRunnerLogger.log(``); + CloudRunnerLogger.log(`Cloud Formation Stacks ${stacks.length}`); for (const element of stacks) { const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime()); @@ -33,13 +37,14 @@ export class TaskService { ageDate.getHours() / 24, )} H${ageDate.getHours()} M${ageDate.getMinutes()}`, ); - if (perResultCallback) await perResultCallback(element); + result.push(element); } const baseStacks = (await CF.listStacks().promise()).StackSummaries?.filter( (_x) => _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription === BaseStackFormation.baseStackDecription, ) || []; + CloudRunnerLogger.log(``); CloudRunnerLogger.log(`Base Stacks ${baseStacks.length}`); for (const element of baseStacks) { const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime()); @@ -49,18 +54,20 @@ export class TaskService { ageDate.getHours() / 24, )} H${ageDate.getHours()} M${ageDate.getMinutes()}`, ); - if (perResultCallback) await perResultCallback(element); - } - if (stacks === undefined) { - return; + result.push(element); } + CloudRunnerLogger.log(``); + + return result; } - public static async awsListTasks(perResultCallback: any = false) { + public static async getTasks() { + const result: any[] = []; + CloudRunnerLogger.log(``); CloudRunnerLogger.log(`List Tasks`); process.env.AWS_REGION = Input.region; const ecs = new AWS.ECS(); const clusters = (await ecs.listClusters().promise()).clusterArns || []; - CloudRunnerLogger.log(`Clusters ${clusters.length}`); + CloudRunnerLogger.log(`Task Clusters ${clusters.length}`); for (const element of clusters) { const input: AWS.ECS.ListTasksRequest = { cluster: element, @@ -71,6 +78,7 @@ export class TaskService { const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element }; const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || []; if (describeList.length === 0) { + CloudRunnerLogger.log(`No Tasks`); continue; } CloudRunnerLogger.log(`Tasks ${describeList.length}`); @@ -84,31 +92,13 @@ export class TaskService { CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`); continue; } - if (perResultCallback) await perResultCallback(taskElement, element); + result.push({ taskElement, element }); } } } - } - public static async awsListJobs(perResultCallback: any = false) { - CloudRunnerLogger.log(`List Jobs`); - process.env.AWS_REGION = Input.region; - const CF = new AWS.CloudFormation(); - const stacks = - (await CF.listStacks().promise()).StackSummaries?.filter( - (_x) => - _x.StackStatus !== 'DELETE_COMPLETE' && _x.TemplateDescription !== BaseStackFormation.baseStackDecription, - ) || []; - CloudRunnerLogger.log(`Stacks ${stacks.length}`); - for (const element of stacks) { - const ageDate: Date = new Date(Date.now() - element.CreationTime.getTime()); + CloudRunnerLogger.log(``); - CloudRunnerLogger.log( - `Task Stack ${element.StackName} - Age D${Math.floor( - ageDate.getHours() / 24, - )} H${ageDate.getHours()} M${ageDate.getMinutes()}`, - ); - if (perResultCallback) await perResultCallback(element); - } + return result; } public static async awsDescribeJob(job: string) { process.env.AWS_REGION = Input.region; @@ -131,4 +121,47 @@ export class TaskService { return message; } + public static async getLogGroups() { + const result: any[] = []; + process.env.AWS_REGION = Input.region; + const ecs = new AWS.CloudWatchLogs(); + let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = { + /* logGroupNamePrefix: 'game-ci' */ + }; + let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise(); + const logGroups = logGroupsDescribe.logGroups || []; + while (logGroupsDescribe.nextToken) { + logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken }; + logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise(); + logGroups.push(...(logGroupsDescribe?.logGroups || [])); + } + + CloudRunnerLogger.log(`Log Groups ${logGroups.length}`); + for (const element of logGroups) { + if (element.creationTime === undefined) { + CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`); + continue; + } + const ageDate: Date = new Date(Date.now() - element.creationTime); + + CloudRunnerLogger.log( + `Task Stack ${element.logGroupName} - Age D${Math.floor( + ageDate.getHours() / 24, + )} H${ageDate.getHours()} M${ageDate.getMinutes()}`, + ); + result.push(element); + } + + return result; + } + public static async getLocks() { + process.env.AWS_REGION = Input.region; + const s3 = new AWS.S3(); + const listRequest: ListObjectsRequest = { + Bucket: `game-ci-test-storage`, + }; + const results = await s3.listObjects(listRequest).promise(); + + return results.Contents || []; + } } diff --git a/src/model/cloud-runner/providers/aws/services/tertiary-resources-service.ts b/src/model/cloud-runner/providers/aws/services/tertiary-resources-service.ts deleted file mode 100644 index 22300269..00000000 --- a/src/model/cloud-runner/providers/aws/services/tertiary-resources-service.ts +++ /dev/null @@ -1,36 +0,0 @@ -import AWS from 'aws-sdk'; -import Input from '../../../../input'; -import CloudRunnerLogger from '../../../services/cloud-runner-logger'; - -export class TertiaryResourcesService { - public static async awsListLogGroups(perResultCallback: any = false) { - process.env.AWS_REGION = Input.region; - const ecs = new AWS.CloudWatchLogs(); - let logStreamInput: AWS.CloudWatchLogs.DescribeLogGroupsRequest = { - /* logGroupNamePrefix: 'game-ci' */ - }; - let logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise(); - const logGroups = logGroupsDescribe.logGroups || []; - while (logGroupsDescribe.nextToken) { - logStreamInput = { /* logGroupNamePrefix: 'game-ci',*/ nextToken: logGroupsDescribe.nextToken }; - logGroupsDescribe = await ecs.describeLogGroups(logStreamInput).promise(); - logGroups.push(...(logGroupsDescribe?.logGroups || [])); - } - - CloudRunnerLogger.log(`Log Groups ${logGroups.length}`); - for (const element of logGroups) { - if (element.creationTime === undefined) { - CloudRunnerLogger.log(`Skipping ${element.logGroupName} no createdAt date`); - continue; - } - const ageDate: Date = new Date(Date.now() - element.creationTime); - - CloudRunnerLogger.log( - `Task Stack ${element.logGroupName} - Age D${Math.floor( - ageDate.getHours() / 24, - )} H${ageDate.getHours()} M${ageDate.getMinutes()}`, - ); - if (perResultCallback) await perResultCallback(element, element); - } - } -} diff --git a/src/model/cloud-runner/remote-client/index.ts b/src/model/cloud-runner/remote-client/index.ts index bfe126fc..e86db9c8 100644 --- a/src/model/cloud-runner/remote-client/index.ts +++ b/src/model/cloud-runner/remote-client/index.ts @@ -105,15 +105,15 @@ export class RemoteClient { } static replaceLargePackageReferencesWithSharedReferences() { - const manifest = fs.readFileSync( - path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`), - 'utf8', - ); - if (CloudRunner.buildParameters.cloudRunnerDebug) { - CloudRunnerLogger.log(manifest); - } if (CloudRunner.buildParameters.useSharedLargePackages) { - manifest.replace(/LargePackages/g, '../../LargePackages'); + const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`); + let manifest = fs.readFileSync(filePath, 'utf8'); + manifest = manifest.replace(/LargeContent/g, '../../LargeContent'); + fs.writeFileSync(filePath, manifest); + if (CloudRunner.buildParameters.cloudRunnerDebug) { + CloudRunnerLogger.log(`Package Manifest`); + CloudRunnerLogger.log(manifest); + } } }