diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 844eca1d..35e2557b 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -12,3 +12,26 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} expire-in: 21 days + cleanupCloudRunner: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + if: github.event.event_type != 'pull_request_target' + with: + lfs: true + - uses: actions/setup-node@v2 + with: + node-version: 12.x + - run: yarn + - run: yarn run cli -m aws-list-tasks + env: + AWS_REGION: eu-west-2 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: eu-west-2 + - run: yarn run cli -m aws-list-stacks + env: + AWS_REGION: eu-west-2 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: eu-west-2 diff --git a/.github/workflows/cloud-runner-aws-pipeline.yml b/.github/workflows/cloud-runner-aws-pipeline.yml index 37e0df24..832e2079 100644 --- a/.github/workflows/cloud-runner-aws-pipeline.yml +++ b/.github/workflows/cloud-runner-aws-pipeline.yml @@ -37,9 +37,9 @@ jobs: #- StandaloneOSX # Build a macOS standalone (Intel 64-bit). - StandaloneWindows64 # Build a Windows 64-bit standalone. - StandaloneLinux64 # Build a Linux 64-bit standalone. + - WebGL # WebGL. #- iOS # Build an iOS player. #- Android # Build an Android .apk. - #- WebGL # WebGL. # - StandaloneWindows # Build a Windows standalone. # - WSAPlayer # Build an Windows Store Apps player. # - PS4 # Build a PS4 Standalone. @@ -91,7 +91,7 @@ jobs: aws s3 ls game-ci-test-storage ls /data/cache/$CACHE_KEY ls /data/cache/$CACHE_KEY/build - aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.zip s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.zip + aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.tar secrets: - name: awsAccessKeyId value: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -100,7 +100,7 @@ jobs: - name: awsDefaultRegion value: eu-west-2 - run: | - aws s3 cp s3://game-ci-test-storage/${{ steps.aws-fargate-unity-build.outputs.CACHE_KEY }}/build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.zip build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.zip + aws s3 cp s3://game-ci-test-storage/${{ steps.aws-fargate-unity-build.outputs.CACHE_KEY }}/build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.tar build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.tar ls - run: yarn run cli -m aws-garbage-collect ########################### @@ -110,5 +110,5 @@ jobs: - uses: actions/upload-artifact@v2 with: name: AWS Build (${{ matrix.targetPlatform }}) - path: build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.zip + path: build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.tar retention-days: 14 diff --git a/.github/workflows/cloud-runner-k8s-pipeline.yml b/.github/workflows/cloud-runner-k8s-pipeline.yml index 1b5b1509..a6e8d11f 100644 --- a/.github/workflows/cloud-runner-k8s-pipeline.yml +++ b/.github/workflows/cloud-runner-k8s-pipeline.yml @@ -105,8 +105,8 @@ jobs: aws s3 ls aws s3 ls game-ci-test-storage ls /data/cache/$CACHE_KEY - echo "/data/cache/$CACHE_KEY/build/build-$BUILD_GUID.zip s3://game-ci-test-storage/$CACHE_KEY/$BUILD_FILE" - aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.zip s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.zip + echo "/data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar s3://game-ci-test-storage/$CACHE_KEY/$BUILD_FILE" + aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.tar s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.tar secrets: - name: awsAccessKeyId value: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -115,7 +115,7 @@ jobs: - name: awsDefaultRegion value: eu-west-2 - run: | - aws s3 cp s3://game-ci-test-storage/${{ steps.k8s-unity-build.outputs.CACHE_KEY }}/build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.zip build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.zip + aws s3 cp s3://game-ci-test-storage/${{ steps.k8s-unity-build.outputs.CACHE_KEY }}/build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.tar build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.tar ls ########################### # Upload # @@ -124,5 +124,5 @@ jobs: - uses: actions/upload-artifact@v2 with: name: K8s Build (${{ matrix.targetPlatform }}) - path: build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.zip + path: build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.tar retention-days: 14 diff --git a/action.yml b/action.yml index 953bd58b..1e0e6a95 100644 --- a/action.yml +++ b/action.yml @@ -115,11 +115,11 @@ inputs: required: false description: 'Either local, k8s or aws can be used to run builds on a remote cluster. Additional parameters must be configured.' cloudRunnerCpu: - default: '1.0' + default: '' required: false description: 'Amount of CPU time to assign the remote build container' cloudRunnerMemory: - default: '750M' + default: '' required: false description: 'Amount of memory to assign the remote build container' cachePushOverrideCommand: diff --git a/dist/cloud-formations/base-setup.yml b/dist/cloud-formations/base-setup.yml deleted file mode 100644 index d5cb05cd..00000000 --- a/dist/cloud-formations/base-setup.yml +++ /dev/null @@ -1,416 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Description: AWS Fargate cluster that can span public and private subnets. Supports - public facing load balancers, private internal load balancers, and - both internal and external service discovery namespaces. -Parameters: - EnvironmentName: - Type: String - Default: development - Description: "Your deployment environment: DEV, QA , PROD" - Version: - Type: String - Description: "hash of template" - - # ContainerPort: - # Type: Number - # Default: 80 - # Description: What port number the application inside the docker container is binding to - - - -Mappings: - # Hard values for the subnet masks. These masks define - # the range of internal IP addresses that can be assigned. - # The VPC can have all IP's from 10.0.0.0 to 10.0.255.255 - # There are four subnets which cover the ranges: - # - # 10.0.0.0 - 10.0.0.255 - # 10.0.1.0 - 10.0.1.255 - # 10.0.2.0 - 10.0.2.255 - # 10.0.3.0 - 10.0.3.255 - - SubnetConfig: - VPC: - CIDR: '10.0.0.0/16' - PublicOne: - CIDR: '10.0.0.0/24' - PublicTwo: - CIDR: '10.0.1.0/24' - -Resources: - - - - # VPC in which containers will be networked. - # It has two public subnets, and two private subnets. - # We distribute the subnets across the first two available subnets - # for the region, for high availability. - VPC: - Type: AWS::EC2::VPC - Properties: - EnableDnsSupport: true - EnableDnsHostnames: true - CidrBlock: !FindInMap ['SubnetConfig', 'VPC', 'CIDR'] - - EFSServerSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupName: "efs-server-endpoints" - GroupDescription: Which client ip addrs are allowed to access EFS server - VpcId: !Ref 'VPC' - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 2049 - ToPort: 2049 - SourceSecurityGroupId: !Ref ContainerSecurityGroup - #CidrIp: !FindInMap ['SubnetConfig', 'VPC', 'CIDR'] - # A security group for the containers we will run in Fargate. - # Rules are added to this security group based on what ingress you - # add for the cluster. - ContainerSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupName: "task security group" - GroupDescription: Access to the Fargate containers - VpcId: !Ref 'VPC' - # SecurityGroupIngress: - # - IpProtocol: tcp - # FromPort: !Ref ContainerPort - # ToPort: !Ref ContainerPort - # CidrIp: 0.0.0.0/0 - SecurityGroupEgress: - - IpProtocol: -1 - FromPort: 2049 - ToPort: 2049 - CidrIp: "0.0.0.0/0" - - - - - # Two public subnets, where containers can have public IP addresses - PublicSubnetOne: - Type: AWS::EC2::Subnet - Properties: - AvailabilityZone: !Select - - 0 - - Fn::GetAZs: !Ref 'AWS::Region' - VpcId: !Ref 'VPC' - CidrBlock: !FindInMap ['SubnetConfig', 'PublicOne', 'CIDR'] - # MapPublicIpOnLaunch: true - - PublicSubnetTwo: - Type: AWS::EC2::Subnet - Properties: - AvailabilityZone: !Select - - 1 - - Fn::GetAZs: !Ref 'AWS::Region' - VpcId: !Ref 'VPC' - CidrBlock: !FindInMap ['SubnetConfig', 'PublicTwo', 'CIDR'] - # MapPublicIpOnLaunch: true - - - # Setup networking resources for the public subnets. Containers - # in the public subnets have public IP addresses and the routing table - # sends network traffic via the internet gateway. - InternetGateway: - Type: AWS::EC2::InternetGateway - GatewayAttachement: - Type: AWS::EC2::VPCGatewayAttachment - Properties: - VpcId: !Ref 'VPC' - InternetGatewayId: !Ref 'InternetGateway' - - # Attaching a Internet Gateway to route table makes it public. - PublicRouteTable: - Type: AWS::EC2::RouteTable - Properties: - VpcId: !Ref 'VPC' - PublicRoute: - Type: AWS::EC2::Route - DependsOn: GatewayAttachement - Properties: - RouteTableId: !Ref 'PublicRouteTable' - DestinationCidrBlock: '0.0.0.0/0' - GatewayId: !Ref 'InternetGateway' - - # Attaching a public route table makes a subnet public. - PublicSubnetOneRouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: !Ref PublicSubnetOne - RouteTableId: !Ref PublicRouteTable - PublicSubnetTwoRouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: !Ref PublicSubnetTwo - RouteTableId: !Ref PublicRouteTable - - - - # ECS Resources - ECSCluster: - Type: AWS::ECS::Cluster - - - - # A role used to allow AWS Autoscaling to inspect stats and adjust scaleable targets - # on your AWS account - AutoscalingRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Service: [application-autoscaling.amazonaws.com] - Action: ['sts:AssumeRole'] - Path: / - Policies: - - PolicyName: service-autoscaling - PolicyDocument: - Statement: - - Effect: Allow - Action: - - 'application-autoscaling:*' - - 'cloudwatch:DescribeAlarms' - - 'cloudwatch:PutMetricAlarm' - - 'ecs:DescribeServices' - - 'ecs:UpdateService' - Resource: '*' - - # This is an IAM role which authorizes ECS to manage resources on your - # account on your behalf, such as updating your load balancer with the - # details of where your containers are, so that traffic can reach your - # containers. - ECSRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Service: [ecs.amazonaws.com] - Action: ['sts:AssumeRole'] - Path: / - Policies: - - PolicyName: ecs-service - PolicyDocument: - Statement: - - Effect: Allow - Action: - # Rules which allow ECS to attach network interfaces to instances - # on your behalf in order for awsvpc networking mode to work right - - 'ec2:AttachNetworkInterface' - - 'ec2:CreateNetworkInterface' - - 'ec2:CreateNetworkInterfacePermission' - - 'ec2:DeleteNetworkInterface' - - 'ec2:DeleteNetworkInterfacePermission' - - 'ec2:Describe*' - - 'ec2:DetachNetworkInterface' - - # Rules which allow ECS to update load balancers on your behalf - # with the information sabout how to send traffic to your containers - - 'elasticloadbalancing:DeregisterInstancesFromLoadBalancer' - - 'elasticloadbalancing:DeregisterTargets' - - 'elasticloadbalancing:Describe*' - - 'elasticloadbalancing:RegisterInstancesWithLoadBalancer' - - 'elasticloadbalancing:RegisterTargets' - Resource: '*' - - # This is a role which is used by the ECS tasks themselves. - ECSTaskExecutionRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Service: [ecs-tasks.amazonaws.com] - Action: ['sts:AssumeRole'] - Path: / - Policies: - - PolicyName: AmazonECSTaskExecutionRolePolicy - PolicyDocument: - Statement: - - Effect: Allow - Action: - - # Allow the use of secret manager - - 'secretsmanager:GetSecretValue' - - 'kms:Decrypt' - - # Allow the ECS Tasks to download images from ECR - - 'ecr:GetAuthorizationToken' - - 'ecr:BatchCheckLayerAvailability' - - 'ecr:GetDownloadUrlForLayer' - - 'ecr:BatchGetImage' - - # Allow the ECS tasks to upload logs to CloudWatch - - 'logs:CreateLogStream' - - 'logs:PutLogEvents' - Resource: '*' - - DeleteCFNLambdaExecutionRole: - Type: "AWS::IAM::Role" - Properties: - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Principal: - Service: ["lambda.amazonaws.com"] - Action: "sts:AssumeRole" - Path: "/" - Policies: - - PolicyName: DeleteCFNLambdaExecutionRole - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Action: - - "logs:CreateLogGroup" - - "logs:CreateLogStream" - - "logs:PutLogEvents" - Resource: "arn:aws:logs:*:*:*" - - Effect: "Allow" - Action: - - "cloudformation:DeleteStack" - - "kinesis:DeleteStream" - - "secretsmanager:DeleteSecret" - - "kinesis:DescribeStreamSummary" - - "logs:DeleteLogGroup" - - "logs:DeleteSubscriptionFilter" - - "ecs:DeregisterTaskDefinition" - - "lambda:DeleteFunction" - - "lambda:InvokeFunction" - - "events:RemoveTargets" - - "events:DeleteRule" - - "lambda:RemovePermission" - Resource: "*" - -### cloud watch to kinesis role - - CloudWatchIAMRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Statement: - - Effect: Allow - Principal: - Service: [logs.amazonaws.com] - Action: ['sts:AssumeRole'] - Path: / - Policies: - - PolicyName: service-autoscaling - PolicyDocument: - Statement: - - Effect: Allow - Action: - - 'kinesis:PutRecord' - Resource: '*' -#####################EFS##################### - - EfsFileStorage: - Type: 'AWS::EFS::FileSystem' - Properties: - BackupPolicy: - Status: ENABLED - PerformanceMode: maxIO - Encrypted: false - - - FileSystemPolicy: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Action: - - "elasticfilesystem:ClientMount" - - "elasticfilesystem:ClientWrite" - - "elasticfilesystem:ClientRootAccess" - Principal: - AWS: "*" - - - MountTargetResource1: - Type: AWS::EFS::MountTarget - Properties: - FileSystemId: !Ref EfsFileStorage - SubnetId: !Ref PublicSubnetOne - SecurityGroups: - - !Ref EFSServerSecurityGroup - - MountTargetResource2: - Type: AWS::EFS::MountTarget - Properties: - FileSystemId: !Ref EfsFileStorage - SubnetId: !Ref PublicSubnetTwo - SecurityGroups: - - !Ref EFSServerSecurityGroup - - - - - - - - -Outputs: - - EfsFileStorageId: - Description: 'The connection endpoint for the database.' - Value: !Ref EfsFileStorage - Export: - Name: !Sub ${EnvironmentName}:EfsFileStorageId - ClusterName: - Description: The name of the ECS cluster - Value: !Ref 'ECSCluster' - Export: - Name: !Sub ${EnvironmentName}:ClusterName - AutoscalingRole: - Description: The ARN of the role used for autoscaling - Value: !GetAtt 'AutoscalingRole.Arn' - Export: - Name: !Sub ${EnvironmentName}:AutoscalingRole - ECSRole: - Description: The ARN of the ECS role - Value: !GetAtt 'ECSRole.Arn' - Export: - Name: !Sub ${EnvironmentName}:ECSRole - ECSTaskExecutionRole: - Description: The ARN of the ECS role tsk execution role - Value: !GetAtt 'ECSTaskExecutionRole.Arn' - Export: - Name: !Sub ${EnvironmentName}:ECSTaskExecutionRole - - DeleteCFNLambdaExecutionRole: - Description: Lambda execution role for cleaning up cloud formations - Value: !GetAtt 'DeleteCFNLambdaExecutionRole.Arn' - Export: - Name: !Sub ${EnvironmentName}:DeleteCFNLambdaExecutionRole - - CloudWatchIAMRole: - Description: The ARN of the CloudWatch role for subscription filter - Value: !GetAtt 'CloudWatchIAMRole.Arn' - Export: - Name: !Sub ${EnvironmentName}:CloudWatchIAMRole - VpcId: - Description: The ID of the VPC that this stack is deployed in - Value: !Ref 'VPC' - Export: - Name: !Sub ${EnvironmentName}:VpcId - PublicSubnetOne: - Description: Public subnet one - Value: !Ref 'PublicSubnetOne' - Export: - Name: !Sub ${EnvironmentName}:PublicSubnetOne - PublicSubnetTwo: - Description: Public subnet two - Value: !Ref 'PublicSubnetTwo' - Export: - Name: !Sub ${EnvironmentName}:PublicSubnetTwo - - ContainerSecurityGroup: - Description: A security group used to allow Fargate containers to receive traffic - Value: !Ref 'ContainerSecurityGroup' - Export: - Name: !Sub ${EnvironmentName}:ContainerSecurityGroup diff --git a/dist/cloud-formations/task-def-formation.yml b/dist/cloud-formations/task-def-formation.yml deleted file mode 100644 index 79e783f8..00000000 --- a/dist/cloud-formations/task-def-formation.yml +++ /dev/null @@ -1,221 +0,0 @@ -AWSTemplateFormatVersion: 2010-09-09 -Description: >- - AWS Fargate cluster that can span public and private subnets. Supports public - facing load balancers, private internal load balancers, and both internal and - external service discovery namespaces. -Parameters: - EnvironmentName: - Type: String - Default: development - Description: 'Your deployment environment: DEV, QA , PROD' - ServiceName: - Type: String - Default: example - Description: A name for the service - ImageUrl: - Type: String - Default: nginx - Description: >- - The url of a docker image that contains the application process that will - handle the traffic for this service - ContainerPort: - Type: Number - Default: 80 - Description: What port number the application inside the docker container is binding to - ContainerCpu: - Type: Number - Default: 1024 - Description: How much CPU to give the container. 1024 is 1 CPU - ContainerMemory: - Type: Number - Default: 2048 - Description: How much memory in megabytes to give the container - BUILDGUID: - Type: String - Default: '' - Command: - Type: String - Default: 'ls' - EntryPoint: - Type: String - Default: '/bin/sh' - WorkingDirectory: - Type: String - Default: '/efsdata/' - Role: - Type: String - Default: '' - Description: >- - (Optional) An IAM role to give the service's containers if the code within - needs to access other AWS resources - EFSMountDirectory: - Type: String - Default: '/efsdata' - # template secrets p1 - input -Mappings: - SubnetConfig: - VPC: - CIDR: 10.0.0.0/16 - PublicOne: - CIDR: 10.0.0.0/24 - PublicTwo: - CIDR: 10.0.1.0/24 -Conditions: - HasCustomRole: !Not - - !Equals - - Ref: Role - - '' -Resources: - LogGroup: - Type: 'AWS::Logs::LogGroup' - Properties: - LogGroupName: !Ref ServiceName - Metadata: - 'AWS::CloudFormation::Designer': - id: aece53ae-b82d-4267-bc16-ed964b05db27 - SubscriptionFilter: - Type: 'AWS::Logs::SubscriptionFilter' - Properties: - FilterPattern: '' - RoleArn: - 'Fn::ImportValue': !Sub '${EnvironmentName}:CloudWatchIAMRole' - LogGroupName: !Ref ServiceName - DestinationArn: - 'Fn::GetAtt': - - KinesisStream - - Arn - Metadata: - 'AWS::CloudFormation::Designer': - id: 7f809e91-9e5d-4678-98c1-c5085956c480 - DependsOn: - - LogGroup - - KinesisStream - KinesisStream: - Type: 'AWS::Kinesis::Stream' - Properties: - Name: !Ref ServiceName - ShardCount: 1 - Metadata: - 'AWS::CloudFormation::Designer': - id: c6f18447-b879-4696-8873-f981b2cedd2b - - # template secrets p2 - secret - - TaskDefinition: - Type: 'AWS::ECS::TaskDefinition' - Properties: - Family: !Ref ServiceName - Cpu: !Ref ContainerCpu - Memory: !Ref ContainerMemory - NetworkMode: awsvpc - Volumes: - - Name: efs-data - EFSVolumeConfiguration: - FilesystemId: - 'Fn::ImportValue': !Sub '${EnvironmentName}:EfsFileStorageId' - TransitEncryption: ENABLED - RequiresCompatibilities: - - FARGATE - ExecutionRoleArn: - 'Fn::ImportValue': !Sub '${EnvironmentName}:ECSTaskExecutionRole' - TaskRoleArn: - 'Fn::If': - - HasCustomRole - - !Ref Role - - !Ref 'AWS::NoValue' - ContainerDefinitions: - - Name: !Ref ServiceName - Cpu: !Ref ContainerCpu - Memory: !Ref ContainerMemory - Image: !Ref ImageUrl - EntryPoint: - Fn::Split: - - "," - - !Ref EntryPoint - Command: - Fn::Split: - - "," - - !Ref Command - WorkingDirectory: !Ref WorkingDirectory - Environment: - - Name: ALLOW_EMPTY_PASSWORD - Value: 'yes' - # template - env vars - MountPoints: - - SourceVolume: efs-data - ContainerPath: !Ref EFSMountDirectory - ReadOnly: false - Secrets: - # template secrets p3 - container def - LogConfiguration: - LogDriver: awslogs - Options: - awslogs-group: !Ref ServiceName - awslogs-region: !Ref 'AWS::Region' - awslogs-stream-prefix: !Ref ServiceName - Metadata: - 'AWS::CloudFormation::Designer': - id: dabb0116-abe0-48a6-a8af-cf9111c879a5 - DependsOn: - - LogGroup -Metadata: - 'AWS::CloudFormation::Designer': - dabb0116-abe0-48a6-a8af-cf9111c879a5: - size: - width: 60 - height: 60 - position: - x: 270 - 'y': 90 - z: 1 - embeds: [] - dependson: - - aece53ae-b82d-4267-bc16-ed964b05db27 - c6f18447-b879-4696-8873-f981b2cedd2b: - size: - width: 60 - height: 60 - position: - x: 270 - 'y': 210 - z: 1 - embeds: [] - 7f809e91-9e5d-4678-98c1-c5085956c480: - size: - width: 60 - height: 60 - position: - x: 60 - 'y': 300 - z: 1 - embeds: [] - dependson: - - aece53ae-b82d-4267-bc16-ed964b05db27 - - c6f18447-b879-4696-8873-f981b2cedd2b - aece53ae-b82d-4267-bc16-ed964b05db27: - size: - width: 150 - height: 150 - position: - x: 60 - 'y': 90 - z: 1 - embeds: [] - 4d2da56c-3643-46b8-aaee-e46e19f95fcc: - source: - id: 7f809e91-9e5d-4678-98c1-c5085956c480 - target: - id: aece53ae-b82d-4267-bc16-ed964b05db27 - z: 11 - 14eb957b-f094-4653-93c4-77b2f851953c: - source: - id: 7f809e91-9e5d-4678-98c1-c5085956c480 - target: - id: c6f18447-b879-4696-8873-f981b2cedd2b - z: 12 - 85c57444-e5bb-4230-bc85-e545cd4558f6: - source: - id: dabb0116-abe0-48a6-a8af-cf9111c879a5 - target: - id: aece53ae-b82d-4267-bc16-ed964b05db27 - z: 13 diff --git a/dist/index.js b/dist/index.js index f577595c..476b3bcd 100644 Binary files a/dist/index.js and b/dist/index.js differ diff --git a/dist/index.js.map b/dist/index.js.map index 1c9e74e8..a1617828 100644 Binary files a/dist/index.js.map and b/dist/index.js.map differ diff --git a/src/model/cloud-runner/cloud-runner-statics.ts b/src/model/cloud-runner/cloud-runner-statics.ts index 964a8980..afebab9b 100644 --- a/src/model/cloud-runner/cloud-runner-statics.ts +++ b/src/model/cloud-runner/cloud-runner-statics.ts @@ -1,3 +1,3 @@ export class CloudRunnerStatics { - public static readonly logPrefix = `Cloud-Runner-System`; + public static readonly logPrefix = `Cloud-Runner`; } diff --git a/src/model/cloud-runner/providers/aws/aws-base-stack.ts b/src/model/cloud-runner/providers/aws/aws-base-stack.ts index 74ee166c..a60cc43a 100644 --- a/src/model/cloud-runner/providers/aws/aws-base-stack.ts +++ b/src/model/cloud-runner/providers/aws/aws-base-stack.ts @@ -1,8 +1,7 @@ import CloudRunnerLogger from '../../services/cloud-runner-logger'; import * as core from '@actions/core'; import * as SDK from 'aws-sdk'; -import * as fs from 'fs'; -import path from 'path'; +import { BaseStackFormation } from './cloud-formations/base-stack-formation'; const crypto = require('crypto'); export class AWSBaseStack { @@ -14,7 +13,7 @@ export class AWSBaseStack { async setupBaseStack(CF: SDK.CloudFormation) { const baseStackName = this.baseStackName; - const baseStack = fs.readFileSync(path.join(__dirname, 'cloud-formations', 'base-setup.yml'), 'utf8'); + const baseStack = BaseStackFormation.formation; // Cloud Formation Input const describeStackInput: SDK.CloudFormation.DescribeStacksInput = { diff --git a/src/model/cloud-runner/providers/aws/aws-cloud-formation-templates.ts b/src/model/cloud-runner/providers/aws/aws-cloud-formation-templates.ts index 91f5a6c7..0d0f83a4 100644 --- a/src/model/cloud-runner/providers/aws/aws-cloud-formation-templates.ts +++ b/src/model/cloud-runner/providers/aws/aws-cloud-formation-templates.ts @@ -1,4 +1,4 @@ -import * as fs from 'fs'; +import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation'; export class AWSCloudFormationTemplates { public static getParameterTemplate(p1) { @@ -34,6 +34,6 @@ export class AWSCloudFormationTemplates { } public static readTaskCloudFormationTemplate(): string { - return fs.readFileSync(`${__dirname}/cloud-formations/task-def-formation.yml`, 'utf8'); + return TaskDefinitionFormation.formation; } } diff --git a/src/model/cloud-runner/providers/aws/aws-job-stack.ts b/src/model/cloud-runner/providers/aws/aws-job-stack.ts index 056d6d6a..c970faa9 100644 --- a/src/model/cloud-runner/providers/aws/aws-job-stack.ts +++ b/src/model/cloud-runner/providers/aws/aws-job-stack.ts @@ -4,6 +4,7 @@ import CloudRunnerSecret from '../../services/cloud-runner-secret'; import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates'; import CloudRunnerLogger from '../../services/cloud-runner-logger'; import { AWSError } from './aws-error'; +import CloudRunner from '../../cloud-runner'; export class AWSJobStack { private baseStackName: string; @@ -23,6 +24,20 @@ export class AWSJobStack { ): Promise { const taskDefStackName = `${this.baseStackName}-${buildGuid}`; let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate(); + const cpu = CloudRunner.buildParameters.cloudRunnerCpu || '1024'; + const memory = CloudRunner.buildParameters.cloudRunnerMemory || '3072'; + taskDefCloudFormation = taskDefCloudFormation.replace( + `ContainerCpu: + Default: 1024`, + `ContainerCpu: + Default: ${Number.parseInt(cpu)}`, + ); + taskDefCloudFormation = taskDefCloudFormation.replace( + `ContainerMemory: + Default: 2048`, + `ContainerMemory: + Default: ${Number.parseInt(memory)}`, + ); for (const secret of secrets) { secret.ParameterKey = `${buildGuid.replace(/[^\dA-Za-z]/g, '')}${secret.ParameterKey.replace( /[^\dA-Za-z]/g, @@ -85,7 +100,9 @@ export class AWSJobStack { }, ...secretsMappedToCloudFormationParameters, ]; - + CloudRunnerLogger.log( + `Starting AWS job with memory: ${CloudRunner.buildParameters.cloudRunnerMemory} cpu: ${CloudRunner.buildParameters.cloudRunnerCpu}`, + ); let previousStackExists = true; while (previousStackExists) { previousStackExists = false; @@ -101,14 +118,15 @@ export class AWSJobStack { } } } + const createStackInput: SDK.CloudFormation.CreateStackInput = { + StackName: taskDefStackName, + TemplateBody: taskDefCloudFormation, + Capabilities: ['CAPABILITY_IAM'], + Parameters: parameters, + }; try { - await CF.createStack({ - StackName: taskDefStackName, - TemplateBody: taskDefCloudFormation, - Capabilities: ['CAPABILITY_IAM'], - Parameters: parameters, - }).promise(); + await CF.createStack(createStackInput).promise(); CloudRunnerLogger.log('Creating cloud runner job'); await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise(); } catch (error) { diff --git a/src/model/cloud-runner/providers/aws/aws-task-runner.ts b/src/model/cloud-runner/providers/aws/aws-task-runner.ts index 772c2644..8b88909b 100644 --- a/src/model/cloud-runner/providers/aws/aws-task-runner.ts +++ b/src/model/cloud-runner/providers/aws/aws-task-runner.ts @@ -58,14 +58,21 @@ class AWSTaskRunner { CloudRunnerLogger.log( `Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(ECS, cluster, taskArn))?.lastStatus}`, ); - const output = await this.streamLogsUntilTaskStops(ECS, CF, taskDef, cluster, taskArn, streamName); + const { output, shouldCleanup } = await this.streamLogsUntilTaskStops( + ECS, + CF, + taskDef, + cluster, + taskArn, + streamName, + ); const taskData = await AWSTaskRunner.describeTasks(ECS, cluster, taskArn); const exitCode = taskData.containers?.[0].exitCode; const wasSuccessful = exitCode === 0 || (exitCode === undefined && taskData.lastStatus === 'RUNNING'); if (wasSuccessful) { CloudRunnerLogger.log(`Cloud runner job has finished successfully`); - return output; + return { output, shouldCleanup }; } else { if (taskData.stoppedReason === 'Essential container in task exited' && exitCode === 1) { throw new Error('Container exited with code 1'); @@ -122,22 +129,24 @@ class AWSTaskRunner { const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${CF.config.region}#logsV2:log-groups/log-group/${taskDef.taskDefStackName}`; CloudRunnerLogger.log(`You can also see the logs at AWS Cloud Watch: ${logBaseUrl}`); let shouldReadLogs = true; + let shouldCleanup = true; let timestamp: number = 0; let output = ''; while (shouldReadLogs) { await new Promise((resolve) => setTimeout(resolve, 1500)); const taskData = await AWSTaskRunner.describeTasks(ECS, clusterName, taskArn); ({ timestamp, shouldReadLogs } = AWSTaskRunner.checkStreamingShouldContinue(taskData, timestamp, shouldReadLogs)); - ({ iterator, shouldReadLogs, output } = await AWSTaskRunner.handleLogStreamIteration( + ({ iterator, shouldReadLogs, output, shouldCleanup } = await AWSTaskRunner.handleLogStreamIteration( kinesis, iterator, shouldReadLogs, taskDef, output, + shouldCleanup, )); } - return output; + return { output, shouldCleanup }; } private static async handleLogStreamIteration( @@ -146,6 +155,7 @@ class AWSTaskRunner { shouldReadLogs: boolean, taskDef: CloudRunnerAWSTaskDef, output: string, + shouldCleanup: boolean, ) { const records = await kinesis .getRecords({ @@ -153,9 +163,16 @@ class AWSTaskRunner { }) .promise(); iterator = records.NextShardIterator || ''; - ({ shouldReadLogs, output } = AWSTaskRunner.logRecords(records, iterator, taskDef, shouldReadLogs, output)); + ({ shouldReadLogs, output, shouldCleanup } = AWSTaskRunner.logRecords( + records, + iterator, + taskDef, + shouldReadLogs, + output, + shouldCleanup, + )); - return { iterator, shouldReadLogs, output }; + return { iterator, shouldReadLogs, output, shouldCleanup }; } private static checkStreamingShouldContinue(taskData: AWS.ECS.Task, timestamp: number, shouldReadLogs: boolean) { @@ -183,6 +200,7 @@ class AWSTaskRunner { taskDef: CloudRunnerAWSTaskDef, shouldReadLogs: boolean, output: string, + shouldCleanup: boolean, ) { if (records.Records.length > 0 && iterator) { for (let index = 0; index < records.Records.length; index++) { @@ -197,11 +215,18 @@ class AWSTaskRunner { shouldReadLogs = false; } else if (message.includes('Rebuilding Library because the asset database could not be found!')) { core.warning('LIBRARY NOT FOUND!'); + core.setOutput('library-found', 'false'); } else if (message.includes('Build succeeded')) { core.setOutput('build-result', 'success'); } else if (message.includes('Build fail')) { core.setOutput('build-result', 'failed'); + core.setFailed('unity build failed'); core.error('BUILD FAILED!'); + } else if (message.includes(': Listening for Jobs')) { + core.setOutput('cloud runner stop watching', 'true'); + shouldReadLogs = false; + shouldCleanup = false; + core.warning('cloud runner stop watching'); } message = `[${CloudRunnerStatics.logPrefix}] ${message}`; if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) { @@ -213,7 +238,7 @@ class AWSTaskRunner { } } - return { shouldReadLogs, output }; + return { shouldReadLogs, output, shouldCleanup }; } private static async getLogStream(kinesis: AWS.Kinesis, kinesisStreamName: string) { diff --git a/src/model/cloud-runner/providers/aws/cloud-formations/base-setup.yml b/src/model/cloud-runner/providers/aws/cloud-formations/base-stack-formation.ts similarity index 93% rename from src/model/cloud-runner/providers/aws/cloud-formations/base-setup.yml rename to src/model/cloud-runner/providers/aws/cloud-formations/base-stack-formation.ts index 3941aeb0..4edd4047 100644 --- a/src/model/cloud-runner/providers/aws/cloud-formations/base-setup.yml +++ b/src/model/cloud-runner/providers/aws/cloud-formations/base-stack-formation.ts @@ -1,7 +1,6 @@ -AWSTemplateFormatVersion: '2010-09-09' -Description: AWS Fargate cluster that can span public and private subnets. Supports - public facing load balancers, private internal load balancers, and - both internal and external service discovery namespaces. +export class BaseStackFormation { + public static readonly formation: string = `AWSTemplateFormatVersion: '2010-09-09' +Description: Game-CI base stack Parameters: EnvironmentName: Type: String @@ -335,57 +334,58 @@ Outputs: Description: 'The connection endpoint for the database.' Value: !Ref EfsFileStorage Export: - Name: !Sub ${EnvironmentName}:EfsFileStorageId + Name: !Sub ${'${EnvironmentName}'}:EfsFileStorageId ClusterName: Description: The name of the ECS cluster Value: !Ref 'ECSCluster' Export: - Name: !Sub ${EnvironmentName}:ClusterName + Name: !Sub${' ${EnvironmentName}'}:ClusterName AutoscalingRole: Description: The ARN of the role used for autoscaling Value: !GetAtt 'AutoscalingRole.Arn' Export: - Name: !Sub ${EnvironmentName}:AutoscalingRole + Name: !Sub ${'${EnvironmentName}'}:AutoscalingRole ECSRole: Description: The ARN of the ECS role Value: !GetAtt 'ECSRole.Arn' Export: - Name: !Sub ${EnvironmentName}:ECSRole + Name: !Sub ${'${EnvironmentName}'}:ECSRole ECSTaskExecutionRole: Description: The ARN of the ECS role tsk execution role Value: !GetAtt 'ECSTaskExecutionRole.Arn' Export: - Name: !Sub ${EnvironmentName}:ECSTaskExecutionRole + Name: !Sub ${'${EnvironmentName}'}:ECSTaskExecutionRole DeleteCFNLambdaExecutionRole: Description: Lambda execution role for cleaning up cloud formations Value: !GetAtt 'DeleteCFNLambdaExecutionRole.Arn' Export: - Name: !Sub ${EnvironmentName}:DeleteCFNLambdaExecutionRole + Name: !Sub ${'${EnvironmentName}'}:DeleteCFNLambdaExecutionRole CloudWatchIAMRole: Description: The ARN of the CloudWatch role for subscription filter Value: !GetAtt 'CloudWatchIAMRole.Arn' Export: - Name: !Sub ${EnvironmentName}:CloudWatchIAMRole + Name: !Sub ${'${EnvironmentName}'}:CloudWatchIAMRole VpcId: Description: The ID of the VPC that this stack is deployed in Value: !Ref 'VPC' Export: - Name: !Sub ${EnvironmentName}:VpcId + Name: !Sub ${'${EnvironmentName}'}:VpcId PublicSubnetOne: Description: Public subnet one Value: !Ref 'PublicSubnetOne' Export: - Name: !Sub ${EnvironmentName}:PublicSubnetOne + Name: !Sub ${'${EnvironmentName}'}:PublicSubnetOne PublicSubnetTwo: Description: Public subnet two Value: !Ref 'PublicSubnetTwo' Export: - Name: !Sub ${EnvironmentName}:PublicSubnetTwo - + Name: !Sub ${'${EnvironmentName}'}:PublicSubnetTwo ContainerSecurityGroup: Description: A security group used to allow Fargate containers to receive traffic Value: !Ref 'ContainerSecurityGroup' Export: - Name: !Sub ${EnvironmentName}:ContainerSecurityGroup + Name: !Sub ${'${EnvironmentName}'}:ContainerSecurityGroup +`; +} diff --git a/src/model/cloud-runner/providers/aws/cloud-formations/task-def-formation.yml b/src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts similarity index 69% rename from src/model/cloud-runner/providers/aws/cloud-formations/task-def-formation.yml rename to src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts index fae6cf4e..49669c4e 100644 --- a/src/model/cloud-runner/providers/aws/cloud-formations/task-def-formation.yml +++ b/src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts @@ -1,4 +1,5 @@ -AWSTemplateFormatVersion: 2010-09-09 +export class TaskDefinitionFormation { + public static readonly formation: string = `AWSTemplateFormatVersion: 2010-09-09 Description: >- AWS Fargate cluster that can span public and private subnets. Supports public facing load balancers, private internal load balancers, and both internal and @@ -23,12 +24,12 @@ Parameters: Default: 80 Description: What port number the application inside the docker container is binding to ContainerCpu: - Type: Number Default: 1024 + Type: Number Description: How much CPU to give the container. 1024 is 1 CPU ContainerMemory: - Type: Number Default: 2048 + Type: Number Description: How much memory in megabytes to give the container BUILDGUID: Type: String @@ -78,7 +79,7 @@ Resources: Properties: FilterPattern: '' RoleArn: - 'Fn::ImportValue': !Sub '${EnvironmentName}:CloudWatchIAMRole' + 'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:CloudWatchIAMRole' LogGroupName: !Ref ServiceName DestinationArn: 'Fn::GetAtt': @@ -98,9 +99,7 @@ Resources: Metadata: 'AWS::CloudFormation::Designer': id: c6f18447-b879-4696-8873-f981b2cedd2b - # template secrets p2 - secret - TaskDefinition: Type: 'AWS::ECS::TaskDefinition' Properties: @@ -112,12 +111,12 @@ Resources: - Name: efs-data EFSVolumeConfiguration: FilesystemId: - 'Fn::ImportValue': !Sub '${EnvironmentName}:EfsFileStorageId' + 'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:EfsFileStorageId' TransitEncryption: ENABLED RequiresCompatibilities: - FARGATE ExecutionRoleArn: - 'Fn::ImportValue': !Sub '${EnvironmentName}:ECSTaskExecutionRole' + 'Fn::ImportValue': !Sub '${'${EnvironmentName}'}:ECSTaskExecutionRole' TaskRoleArn: 'Fn::If': - HasCustomRole @@ -153,69 +152,7 @@ Resources: awslogs-group: !Ref ServiceName awslogs-region: !Ref 'AWS::Region' awslogs-stream-prefix: !Ref ServiceName - Metadata: - 'AWS::CloudFormation::Designer': - id: dabb0116-abe0-48a6-a8af-cf9111c879a5 DependsOn: - LogGroup -Metadata: - 'AWS::CloudFormation::Designer': - dabb0116-abe0-48a6-a8af-cf9111c879a5: - size: - width: 60 - height: 60 - position: - x: 270 - 'y': 90 - z: 1 - embeds: [] - dependson: - - aece53ae-b82d-4267-bc16-ed964b05db27 - c6f18447-b879-4696-8873-f981b2cedd2b: - size: - width: 60 - height: 60 - position: - x: 270 - 'y': 210 - z: 1 - embeds: [] - 7f809e91-9e5d-4678-98c1-c5085956c480: - size: - width: 60 - height: 60 - position: - x: 60 - 'y': 300 - z: 1 - embeds: [] - dependson: - - aece53ae-b82d-4267-bc16-ed964b05db27 - - c6f18447-b879-4696-8873-f981b2cedd2b - aece53ae-b82d-4267-bc16-ed964b05db27: - size: - width: 150 - height: 150 - position: - x: 60 - 'y': 90 - z: 1 - embeds: [] - 4d2da56c-3643-46b8-aaee-e46e19f95fcc: - source: - id: 7f809e91-9e5d-4678-98c1-c5085956c480 - target: - id: aece53ae-b82d-4267-bc16-ed964b05db27 - z: 11 - 14eb957b-f094-4653-93c4-77b2f851953c: - source: - id: 7f809e91-9e5d-4678-98c1-c5085956c480 - target: - id: c6f18447-b879-4696-8873-f981b2cedd2b - z: 12 - 85c57444-e5bb-4230-bc85-e545cd4558f6: - source: - id: dabb0116-abe0-48a6-a8af-cf9111c879a5 - target: - id: aece53ae-b82d-4267-bc16-ed964b05db27 - z: 13 +`; +} diff --git a/src/model/cloud-runner/providers/aws/commands/aws-cli-commands.ts b/src/model/cloud-runner/providers/aws/commands/aws-cli-commands.ts index 3c0d237b..deb6f9ce 100644 --- a/src/model/cloud-runner/providers/aws/commands/aws-cli-commands.ts +++ b/src/model/cloud-runner/providers/aws/commands/aws-cli-commands.ts @@ -4,27 +4,89 @@ import Input from '../../../../input'; import CloudRunnerLogger from '../../../services/cloud-runner-logger'; export class AwsCliCommands { - @CliFunction(`aws-garbage-collect`, `garbage collect aws`) - static async garbageCollectAws() { + @CliFunction(`aws-list-stacks`, `List stacks`) + static async awsListStacks(perResultCallback: any) { process.env.AWS_REGION = Input.region; - CloudRunnerLogger.log(`Cloud Formation stacks`); const CF = new AWS.CloudFormation(); const stacks = (await CF.listStacks().promise()).StackSummaries?.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE') || []; + CloudRunnerLogger.log(`DescribeStacksRequest ${stacks.length}`); for (const element of stacks) { CloudRunnerLogger.log(JSON.stringify(element, undefined, 4)); + CloudRunnerLogger.log(`${element.StackName}`); + if (perResultCallback) await perResultCallback(element); } - CloudRunnerLogger.log(`ECS Clusters`); - const ecs = new AWS.ECS(); - const clusters = (await ecs.listClusters().promise()).clusterArns || []; if (stacks === undefined) { return; } + } + @CliFunction(`aws-list-tasks`, `List tasks`) + static async awsListTasks(perResultCallback: any) { + process.env.AWS_REGION = Input.region; + CloudRunnerLogger.log(`ECS Clusters`); + const ecs = new AWS.ECS(); + const clusters = (await ecs.listClusters().promise()).clusterArns || []; for (const element of clusters) { const input: AWS.ECS.ListTasksRequest = { cluster: element, }; - CloudRunnerLogger.log(JSON.stringify(await ecs.listTasks(input).promise(), undefined, 4)); + const list = (await ecs.listTasks(input).promise()).taskArns || []; + if (list.length > 0) { + const describeInput: AWS.ECS.DescribeTasksRequest = { tasks: list, cluster: element }; + const describeList = (await ecs.describeTasks(describeInput).promise()).tasks || []; + if (describeList === []) { + continue; + } + CloudRunnerLogger.log(`DescribeTasksRequest ${describeList.length}`); + for (const taskElement of describeList) { + if (taskElement === undefined) { + continue; + } + taskElement.overrides = {}; + taskElement.attachments = []; + CloudRunnerLogger.log(JSON.stringify(taskElement, undefined, 4)); + if (taskElement.createdAt === undefined) { + CloudRunnerLogger.log(`Skipping ${taskElement.taskDefinitionArn} no createdAt date`); + continue; + } + if (perResultCallback) await perResultCallback(taskElement, element); + } + } } } + + @CliFunction(`aws-garbage-collect`, `garbage collect aws`) + static async garbageCollectAws() { + await AwsCliCommands.cleanup(false); + } + @CliFunction(`aws-garbage-collect-all`, `garbage collect aws`) + static async garbageCollectAwsAll() { + await AwsCliCommands.cleanup(true); + } + @CliFunction(`aws-garbage-collect-all-1d-older`, `garbage collect aws`) + static async garbageCollectAwsAllOlderThanOneDay() { + await AwsCliCommands.cleanup(true); + } + + private static async cleanup(deleteResources = false) { + process.env.AWS_REGION = Input.region; + const CF = new AWS.CloudFormation(); + const ecs = new AWS.ECS(); + await AwsCliCommands.awsListStacks(async (element) => { + if (deleteResources) { + if (element.StackName === 'game-ci' || element.TemplateDescription === 'Game-CI base stack') { + CloudRunnerLogger.log(`Skipping ${element.StackName} ignore list`); + + return; + } + const deleteStackInput: AWS.CloudFormation.DeleteStackInput = { StackName: element.StackName }; + await CF.deleteStack(deleteStackInput).promise(); + } + }); + await AwsCliCommands.awsListTasks(async (taskElement, element) => { + if (deleteResources) { + await ecs.stopTask({ task: taskElement.taskArn || '', cluster: element }).promise(); + } + }); + } } diff --git a/src/model/cloud-runner/providers/aws/index.ts b/src/model/cloud-runner/providers/aws/index.ts index 4d8fca31..4be2f66f 100644 --- a/src/model/cloud-runner/providers/aws/index.ts +++ b/src/model/cloud-runner/providers/aws/index.ts @@ -66,21 +66,24 @@ class AWSBuildEnvironment implements ProviderInterface { ); let postRunTaskTimeMs; - let output = ''; try { const postSetupStacksTimeMs = Date.now(); CloudRunnerLogger.log(`Setup job time: ${Math.floor((postSetupStacksTimeMs - startTimeMs) / 1000)}s`); - output = await AWSTaskRunner.runTask(taskDef, ECS, CF, environment, buildGuid, commands); + const { output, shouldCleanup } = await AWSTaskRunner.runTask(taskDef, ECS, CF, environment, buildGuid, commands); postRunTaskTimeMs = Date.now(); CloudRunnerLogger.log(`Run job time: ${Math.floor((postRunTaskTimeMs - postSetupStacksTimeMs) / 1000)}s`); - } finally { - await this.cleanupResources(CF, taskDef); + if (shouldCleanup) { + await this.cleanupResources(CF, taskDef); + } const postCleanupTimeMs = Date.now(); if (postRunTaskTimeMs !== undefined) CloudRunnerLogger.log(`Cleanup job time: ${Math.floor((postCleanupTimeMs - postRunTaskTimeMs) / 1000)}s`); - } - return output; + return output; + } catch (error) { + await this.cleanupResources(CF, taskDef); + throw error; + } } async cleanupResources(CF: SDK.CloudFormation, taskDef: CloudRunnerAWSTaskDef) { diff --git a/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts b/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts index 8e7cf026..0bb9294a 100644 --- a/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts +++ b/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts @@ -108,8 +108,8 @@ class KubernetesJobSpecFactory { workingDir: `${workingDirectory}`, resources: { requests: { - memory: buildParameters.cloudRunnerMemory, - cpu: buildParameters.cloudRunnerCpu, + memory: buildParameters.cloudRunnerMemory || '750M', + cpu: buildParameters.cloudRunnerCpu || '1', }, }, env: [ diff --git a/src/model/cloud-runner/remote-client/caching.ts b/src/model/cloud-runner/remote-client/caching.ts index 277353c3..19cdbf03 100644 --- a/src/model/cloud-runner/remote-client/caching.ts +++ b/src/model/cloud-runner/remote-client/caching.ts @@ -70,17 +70,17 @@ export class Caching { return typeof arguments_[number] != 'undefined' ? arguments_[number] : match; }); }; - await CloudRunnerSystem.Run(`zip -q -r ${cacheArtifactName}.zip ${path.basename(sourceFolder)}`); - assert(await fileExists(`${cacheArtifactName}.zip`), 'cache zip exists'); + await CloudRunnerSystem.Run(`tar -cf ${cacheArtifactName}.tar ${path.basename(sourceFolder)}`); + assert(await fileExists(`${cacheArtifactName}.tar`), 'cache archive exists'); assert(await fileExists(path.basename(sourceFolder)), 'source folder exists'); if (CloudRunner.buildParameters.cachePushOverrideCommand) { await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePushOverrideCommand)); } - await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.zip ${cacheFolder}`); - RemoteClientLogger.log(`moved ${cacheArtifactName}.zip to ${cacheFolder}`); + await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.tar ${cacheFolder}`); + RemoteClientLogger.log(`moved cache entry ${cacheArtifactName} to ${cacheFolder}`); assert( - await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.zip`), - 'cache zip exists inside cache folder', + await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.tar`), + 'cache archive exists inside cache folder', ); } catch (error) { process.chdir(`${startPath}`); @@ -101,14 +101,14 @@ export class Caching { await fs.promises.mkdir(destinationFolder); } - const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .zip$ | head -1`)) + const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .tar$ | head -1`)) .replace(/\n/g, ``) - .replace('.zip', ''); + .replace('.tar', ''); process.chdir(cacheFolder); const cacheSelection = - cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.zip`)) ? cacheArtifactName : latestInBranch; + cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.tar`)) ? cacheArtifactName : latestInBranch; await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`); // eslint-disable-next-line func-style @@ -127,12 +127,12 @@ export class Caching { await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePullOverrideCommand)); } - if (await fileExists(`${cacheSelection}.zip`)) { + if (await fileExists(`${cacheSelection}.tar`)) { const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`; await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`); - RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.zip`); + RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.tar`); const fullResultsFolder = path.join(cacheFolder, resultsFolder); - await CloudRunnerSystem.Run(`unzip -q ${cacheSelection}.zip -d ${path.basename(resultsFolder)}`); + await CloudRunnerSystem.Run(`tar -xf ${cacheSelection}.tar -C ${fullResultsFolder}`); RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`); assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`); const destinationParentFolder = path.resolve(destinationFolder, '..'); @@ -143,7 +143,6 @@ export class Caching { await CloudRunnerSystem.Run( `mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`, ); - await CloudRunnerSystem.Run(`du -sh ${path.join(destinationParentFolder, path.basename(destinationFolder))}`); const contents = await fs.promises.readdir( path.join(destinationParentFolder, path.basename(destinationFolder)), ); @@ -153,7 +152,7 @@ export class Caching { } else { RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`); if (cacheSelection !== ``) { - RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.zip doesn't exist ${destinationFolder}`); + RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.tar doesn't exist ${destinationFolder}`); throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`); } } diff --git a/src/model/cloud-runner/remote-client/index.ts b/src/model/cloud-runner/remote-client/index.ts index bf80fbc7..39cfcf70 100644 --- a/src/model/cloud-runner/remote-client/index.ts +++ b/src/model/cloud-runner/remote-client/index.ts @@ -45,8 +45,10 @@ export class RemoteClient { } private static async sizeOfFolder(message: string, folder: string) { - CloudRunnerLogger.log(`Size of ${message}`); - await CloudRunnerSystem.Run(`du -sh ${folder}`); + if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) { + CloudRunnerLogger.log(`Size of ${message}`); + await CloudRunnerSystem.Run(`du -sh ${folder}`); + } } private static async cloneRepoWithoutLFSFiles() { diff --git a/src/model/cloud-runner/workflows/build-automation-workflow.ts b/src/model/cloud-runner/workflows/build-automation-workflow.ts index 3552dff8..ac02ef9a 100644 --- a/src/model/cloud-runner/workflows/build-automation-workflow.ts +++ b/src/model/cloud-runner/workflows/build-automation-workflow.ts @@ -71,7 +71,7 @@ export class BuildAutomationWorkflow implements WorkflowInterface { const builderPath = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`).replace(/\\/g, `/`); return `apt-get update > /dev/null - apt-get install -y zip tree npm git-lfs jq unzip git > /dev/null + apt-get install -y tar tree npm git-lfs jq git > /dev/null npm install -g n > /dev/null n stable > /dev/null ${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '} diff --git a/src/model/input-readers/generic-input-reader.ts b/src/model/input-readers/generic-input-reader.ts index 0bd84e11..85a42fea 100644 --- a/src/model/input-readers/generic-input-reader.ts +++ b/src/model/input-readers/generic-input-reader.ts @@ -1,7 +1,12 @@ import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system'; +import Input from '../input'; export class GenericInputReader { public static async Run(command) { + if (Input.cloudRunnerCluster === 'local') { + return ''; + } + return await CloudRunnerSystem.Run(command, false, true); } } diff --git a/src/model/input-readers/git-repo.ts b/src/model/input-readers/git-repo.ts index c301f5e8..3372089e 100644 --- a/src/model/input-readers/git-repo.ts +++ b/src/model/input-readers/git-repo.ts @@ -2,9 +2,13 @@ import { assert } from 'console'; import fs from 'fs'; import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system'; import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger'; +import Input from '../input'; export class GitRepoReader { public static async GetRemote() { + if (Input.cloudRunnerCluster === 'local') { + return ''; + } assert(fs.existsSync(`.git`)); const value = (await CloudRunnerSystem.Run(`git remote -v`, false, true)).replace(/ /g, ``); CloudRunnerLogger.log(`value ${value}`); @@ -14,6 +18,9 @@ export class GitRepoReader { } public static async GetBranch() { + if (Input.cloudRunnerCluster === 'local') { + return ''; + } assert(fs.existsSync(`.git`)); return (await CloudRunnerSystem.Run(`git branch --show-current`, false, true)) diff --git a/src/model/input-readers/github-cli.ts b/src/model/input-readers/github-cli.ts index 4e9754a2..7ad6ef7a 100644 --- a/src/model/input-readers/github-cli.ts +++ b/src/model/input-readers/github-cli.ts @@ -1,8 +1,12 @@ import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system'; import * as core from '@actions/core'; +import Input from '../input'; export class GithubCliReader { static async GetGitHubAuthToken() { + if (Input.cloudRunnerCluster === 'local') { + return ''; + } try { const authStatus = await CloudRunnerSystem.Run(`gh auth status`, true, true); if (authStatus.includes('You are not logged') || authStatus === '') { diff --git a/src/model/input-readers/test-license-reader.ts b/src/model/input-readers/test-license-reader.ts index b55ce73a..5d072743 100644 --- a/src/model/input-readers/test-license-reader.ts +++ b/src/model/input-readers/test-license-reader.ts @@ -1,8 +1,12 @@ import path from 'path'; import fs from 'fs'; import YAML from 'yaml'; +import Input from '../input'; export function ReadLicense() { + if (Input.cloudRunnerCluster === 'local') { + return ''; + } const pipelineFile = path.join(__dirname, `.github`, `workflows`, `cloud-runner-k8s-pipeline.yml`); return fs.existsSync(pipelineFile) ? YAML.parse(fs.readFileSync(pipelineFile, 'utf8')).env.UNITY_LICENSE : ''; diff --git a/src/model/input.ts b/src/model/input.ts index 36c86c55..d3bb8116 100644 --- a/src/model/input.ts +++ b/src/model/input.ts @@ -234,11 +234,11 @@ class Input { } static get cloudRunnerCpu() { - return Input.getInput('cloudRunnerCpu') || '1.0'; + return Input.getInput('cloudRunnerCpu'); } static get cloudRunnerMemory() { - return Input.getInput('cloudRunnerMemory') || '750M'; + return Input.getInput('cloudRunnerMemory'); } static get kubeConfig() {