From 2e2eae0acf99b118111e21eae6453660bbc9fb4a Mon Sep 17 00:00:00 2001 From: Frostebite Date: Mon, 17 Jul 2023 17:28:16 +0100 Subject: [PATCH] push k8s logs to LOG SERVICE IP --- dist/index.js | Bin 19465808 -> 19472276 bytes dist/index.js.map | Bin 13765665 -> 13773593 bytes src/model/cloud-runner/providers/k8s/index.ts | 28 +++- .../k8s/kubernetes-job-spec-factory.ts | 2 + .../providers/k8s/kubernetes-log-service.ts | 153 ++++++++++++++++++ .../remote-client/remote-client-logger.ts | 1 + 6 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 src/model/cloud-runner/providers/k8s/kubernetes-log-service.ts diff --git a/dist/index.js b/dist/index.js index 8581817d08f07dd8c0f73539a4c0aaf80c7be574..b07d9516bf8573f66a2d0dd49c253c646d07223a 100644 GIT binary patch delta 4736 zcmb`~dvp}l9S88UH=9j13E70*Y#s!169Svi>?WZ=<5CS6!$UzqC{h%Y$?lMh+02YP zlRyeIp;#$Z$|IDYEh1>?!#=F2S z)?ZA6FSBfV?J-s{JXp+TGy7a!H)y1kO|gC(zO$5>wt@P3uJ;;U9K9Ia)VSKIsO$I| zB_eatsT}2fio|`3u9@Gqto7Oz3tL*-7B2B|S?%L}QZ$Fhe?ppSm|W5MrCg6{A=xkj zXmWk~==<7N&raT3^PSX#~n?!fL!u+)VICIvHZqHTC#4yY{ZgeZUuBe@2oYsJ8 z)1f>s?a1O~<-O4e9^2k!ntHIlzIwf)Z>*Per5;V&>5Mbu5-Kle*{;M4tJmX(jqO7P zJsvu6&(2IL*iq`Ft#{e|6(P};e&0CgtGIO&9aS^R=mTd)Jw08{ zT<%y;xE7TS9Sb0DY^R-%+uXw+ma`{|L)L0YYbC|x-Mt-hRFzHHXhVnb?ULIX17UfM z)EhAap{OjG@*=JC3R#b6{XMd3)|g$25m2QbbS{PDI&OUyJ+(v>T#s#MH7lpZdo!|! zPH%P46EoNw_Bw3CkZrens9~3b+UD8DQFDaN94gptcVe?+;~6fV#SXZfqA8Zn=Nlew zWwUaJPEWDZ+c(;sWuv#jSBJfh^vXdrr<P)K3JQkvma>jayEt?Y%{euk zLArg?+@T|{I;h*j8tKjJnCmLcYMFIDWxn3pehkAfB~uA;$>>)@d`+(!G8Ij&Na3w$7pwlIfkzti+k@kWNUfk^(ES1~*M)`NNMynQWs~t+ql}SdL)1&u!@? zGr|ffcMhwdNR&;aw*4$?=y;Et&TeLo{Lwo=(kfOoxz5_Y*rK;+QMoog4RQ1_m75Gr zr4u>#dT3=E%Nvl_nX+nNHb#A4V~1=?jnsok~y*n`Vq$&`Zex5An}Ze6Zm z(@m*I4|26P5{X}pNF8#-NVMgNH3@QY^8MnK#Tz?-S2o^W9G%e@6W0t~4kh}DVxg6g zWCVF^py=0(;w*Od2l@Jq@vcm{M~~o-iJLedJ9M!aV)#?UMxKBCqDg!XbY^e4ho66N zqFHQ!boYy1_dwc=Irnp=N9x41diwogEz~VX{aw8sekl~vdR4PNeI+nif9kypis2h( zd_LkCJl53`EbtNyOMv;#jjWaWjZtk>YBN}7!I+M7Xw#$lo|#y*zD;iU$s-`xT zU{j3)$c8KAs5q43ZI@67nsGRN=WQzvs%Vw!^nC5;28B(af;}!rp>NFE_-N@oR#O%$ zN!Y1#ub(+vQA+Kz%CmO8XU|Culb)Z)yjccb+>ohJ;WOik=sC@5`oAmn9-bRJ%I+Nm1SS#5#Bvd z)}dM~V&iGf0_LWL^O1Lsyx?`xt_5uNW$}aQ``=C`ZJKEF(BPIU@=C0Z(vPJtb}s4g zpH#)T^u=BVW-!xMDed&&WaO*_QsNS#W7d5cg#5*)jy`@qO)f! zauYiw5@sdsdud7zy?b(MUT$i`YU#)wh8MlXN>@C|x31GQL$0xM=jB$uOr0~FdaE8P zDhj!#bG)-kL?kk$FC>d%B!$BmoWT`?%T_;OsvK~*>AsfE@vO0dr1_ntbn57Vyb1B6 zAilF0T~ajt?jhSAx2Sx6>y8MvD!-*Ot0=8|ZA~J3($U?_G9NzT4Q7sltMaM$4Y4APUdK`s%Ly4fetO)ABer@&4Gv}g)f&Pk{Xhu%>jII5wmIz zT;H%7pL9B$o9DLabTm6}0>j6s;|QyC$ZwGyKgvy81rT4A(`VPUew$!QB;nRRYq z>PzHz&SS+WzF)}~<=m8h@p_cZ=E)u|P?M~(Ga%kW{9L#yiL2?ANqp9|s5zzp#5!nd z@HaK2Jvi1h$jAOMs+n3yiv;=fgc3->+LjVI>)O=cw_P!SZ2s1*?0O z7H?pcRTurYhi}-x+qZ11$^biLf&;Q38=Q~>xsV6rARk;%0EI9f+~9#C@Io<^Kq-{L z1Sp4zFbOJvLnTzfWT=KIFco}I1GO*>J_2>%hXB+=12n>P_$bVPnJ^1J22F4Y1YtIO z94>{+pc&@CT(}%s;0kDkc`zRqz(TkZu7X9d7_NqEU!PSxDnbwth)pq5P~qsum(Dz3lz8sx*-BRpn?WE+ze4Lz=U4tgSD^@`r%Ws9yY*6 z_%sZ_AZ&uqz-G7wJ`1+FTi%#0XtzA48d;rBJ2TzJK#(3W!MXM z!aleQ_QL_V8}5Oxz(M#bd=0)1-++7JoA538Hhc%}gYUxq@Blmr55dFm2s{eI@IClG zJO)32AHw7C1RR19I1Eq1Q}82r8h#AVz)#?(@H6;1JPW^o=iqrb0!QHmcoBXHFTu-j z41NX2;n(mAyb8a8*Wh(H0l$Se;3WJGeh+WLAK;I03jPFd!Q1c-{2AVb({Kj<0`I|J z;eGfU{2l%Q|Ac?R2k;^M8~y`lZ`)R7+YTZy5i&%ui;yXTLxe05vPE!;kRw8_2zerm z6Cqy&mk0$S6pAoj1h)tt5sF0cicl;ev!tR!Ik%qnCP(X1%`5Us^eCs?WQ?T|3j+IHuSUr~h-g zd^IhlHO_!jpRBir{^R{ztH)RCx6Wf+ZVsE#U<921uGYQI2E(pbTUrBkZhz@QpKF%0 zq1*_S$hvCV(6GPJIiER06J+|(vrYFnU53YR*IRXGW20TS2EFb;ONlG!_S6_nRz0ML z)YhViiTSgG{yMj}RuA~}89|rP5bEwYRGSM;x~gc`(ht4fQ>?|S_DNbsIJHdcOj7l`a}(4$+a`0kzg)A#tG*V- z+bXpLmAu^?6P{k7xuU`@pJv;&V$czRNJPPmXv82Eafn9(5-|)(NJa{VBNb^#M+P#H zg>2+t1adJFdB}&30u*8tEGWWgSTP30I08q)h7y$GD2&B89F1deERMr?Oh6eX!VU)} zVKS!Rc$|O}aT2EDWSoL&D95Roj?*v$r(-5&;S9{inK%m-I2&_NiF0r+&cpebiwiIh z7orN)aKeT8sDXh6s6`#zScrOf&;T!dXv88k!H)ofxCqTyj22vsR$PJ*mS8E`und>t zGAu_sF2@R7fh%zpR$>*dMh8~o8gya}y3mblaUB${#|^j9D*OL!Rrcm=QGHN1{Dup4jUExe6)@Gkb?Jq+S~e1H$J7yIxLKE{50 zf=}@oKF1e0fG_bCzQ#BB7T@7}{D2?v6Mn`o_!Yn5cl?1raS(s49CVmgL!wC{B$1LR ziCGdYiIK!g;w15s1WBS~m?TM(EJ=|Jm!wM4B?Q1%um7pt*<@Jzvi(x3k85H1jCAg zp~zacG87h>_7E}3xvSh(c%{SC9Xt|-u$;3!E>lUJPE^0SlF0-qd9F+p1 zqPTJ6_SW-Dx30Kyd31S%B|f2*tRf`S;d*;H-MfQ#U3QBQjf5iY6=SNgh@_$qY86vf zhh!Ngv`*^1rMkOpW-7eVW+ZP(^&}d(+ivE&qG9x-wL?{9zsECuT}a;G5#*2um0D^4 zD*h^K69$@SMjz|+b*Qs}_7>QS`(M7ih~8~#RjQ;z9@&K-cB*1R?w!GxUzU8?|X@xMrXXdiJWh6e>7Av z%wqW=jwic;UqIfOj%+G;l`{@)eUn>On5ErjwD;EzGYu@V8Hc|312>XOh3(S}G|by` zZ0Eq~mRK!~*`E#zE8pM_Fg|*w;|Q6Dt5|YT-R&I5?h>&GZ5{TKF60Dj8-Ty zpwuI@i+Fr|iCuIETA7^DaZpC3&m2?I#-!bMu~4Vbj%n!Zbp@rsYBB2S=xTEbfq>K% zQ9WZO$E5{78o{bk{Mrx|b!?yX)|wEGCJFO}nW+o+2)#03ZZJAgG@1T=*-dN-uGf7U`{vPKp9gX?e`?TgU8>6|~xqR&Vw^>b{f*R;(&^8JQd$|(*6ud;^8IRhm{khag9R{AGhQ;$uJ6Ua>DHSj}_hG4p29I(^Ycli2 z4{ve7T3J%W3OZZG7hk8Jyyq=99K1b3NM)f2a<3#aY~D~txO(KG*W}KucfCFYy?5?>G^rYO`A^LQEaIgJLAa0R5yp` zcUH|b&e7hg5`A;GNx@!s{0OLZhb19+jSv}N~so3K2Kj=pJ$HH}+%^f(*ac-k;8I!eaW zUKu&BA#K^Dx0A>$`lVTkQ6dYcmA+`!DHTSS&)Af6c#EUdf)$)fHHnbZ+a~M|!OoY_ zo?Je=$em0WiTh6K%jGRpo`JWs|1&q}56DWTg$W2Ao%?_*rH8-55AD!1I`yE%k3#)K zo42dlm|5+mv!}IkyY@{q`k50EN-Og0pN<gZKLM}abb32iK^yOJyh{)D z+XDJ4`B3&}oaY@nQ)4fmddYvx*!uOXRsRQw-k;59)4}@@n~hgHawz*1-mxuqga
    iznc##9m<5&a4X6SaxWNNn@WEy9O}HFp!xb~4X_Yyga-H)EP_U8g2iwXEP5irLnlN)0vXmo6ckXQ3%a2P)b~ycVG)_g>7&*Y=`f{J@7sFKHLlU!Trz&KY$(ZL+FQ{K=1(Uf*-+yup1tNJ@7CL zz+QL+ehiPoWAHdU0Z+nD;HU5uJPrHc85o3TVLv_+uN*0a z7c=3rGpg7F&UJl}qBr=4fPv zW|qyXY3^(`#NQ)bltQT9#&im_X$dXlkHsUFzlHtj@Sl31Lr-YwB86^YXdQY`w9f5L zv`4LgoU=ytEewjOhxC$VYMb;0BBq7)m=&b2nPyPR#4C|eSp3p7v&&SKbgKB}SpD4?b`zI8aoN%cgIn=k>ng8ztr~a!V zrPj&XVa1*Qzc2fxx+q^Zj4BR!VWm=>zPM@jz_60#lY#Tfq%0X%X2{lWXE|lXn6g}+ z94T^0<1D8(jw^+B-5 z>8+YnQH)ub4S_i*!DX0>%TWqH$}kUCU_KV0906R33RL1MRAC{mMl}{;F|I)nHK;`$ zu0=f>a2>A465N1B+=!)EhUK^kO}H5=uoAam6>i09+=ew+i*;C!4cLg=5kfOsa0l*0 zD>Q_$32o4^84J?O%{=*E5M!Topu+wmYC!VWx) zN3avU*oEDA6p!I?Jb@?i6!xGGPop2t;8{EeiM`l|=P`g6upcktCA^G5yn?rv-od*VhK&)thxhRTKEy}(7^66eQ#g$?IEznk4xb{0 X&+s|Ez?b+6U*j8`x2q<{epvPwi;_+2 diff --git a/src/model/cloud-runner/providers/k8s/index.ts b/src/model/cloud-runner/providers/k8s/index.ts index 83f85163..ec619e42 100644 --- a/src/model/cloud-runner/providers/k8s/index.ts +++ b/src/model/cloud-runner/providers/k8s/index.ts @@ -16,11 +16,14 @@ import { ProviderResource } from '../provider-resource'; import { ProviderWorkflow } from '../provider-workflow'; import { RemoteClientLogger } from '../../remote-client/remote-client-logger'; import { KubernetesRole } from './kubernetes-role'; +import KubernetesLogService from './kubernetes-log-service'; +import { CloudRunnerSystem } from '../../services/core/cloud-runner-system'; class Kubernetes implements ProviderInterface { public static Instance: Kubernetes; public kubeConfig!: k8s.KubeConfig; public kubeClient!: k8s.CoreV1Api; + public kubeClientApps!: k8s.AppsV1Api; public kubeClientBatch!: k8s.BatchV1Api; public rbacAuthorizationV1Api!: k8s.RbacAuthorizationV1Api; public buildGuid: string = ''; @@ -40,6 +43,7 @@ class Kubernetes implements ProviderInterface { this.kubeConfig = new k8s.KubeConfig(); this.kubeConfig.loadFromDefault(); this.kubeClient = this.kubeConfig.makeApiClient(k8s.CoreV1Api); + this.kubeClientApps = this.kubeConfig.makeApiClient(k8s.AppsV1Api); this.kubeClientBatch = this.kubeConfig.makeApiClient(k8s.BatchV1Api); this.rbacAuthorizationV1Api = this.kubeConfig.makeApiClient(k8s.RbacAuthorizationV1Api); this.namespace = 'default'; @@ -47,13 +51,17 @@ class Kubernetes implements ProviderInterface { } async PushLogUpdate(logs: string) { - const body: k8s.V1ConfigMap = new k8s.V1ConfigMap(); - body.data = {}; - body.data['logs'] = logs; - body.metadata = { name: `${this.jobName}-logs`, namespace: this.namespace, labels: { app: 'unity-builder' } }; - RemoteClientLogger.log(`Pushing to Kubernetes ConfigMap`); - await this.kubeClient.createNamespacedConfigMap(this.namespace, body); - RemoteClientLogger.log(`Pushed logs to Kubernetes ConfigMap`); + // push logs to nginx file server via 'LOG_SERVICE_IP' env var + const ip = process.env[`LOG_SERVICE_IP`]; + if (ip === undefined) { + RemoteClientLogger.logWarning(`LOG_SERVICE_IP not set, skipping log push`); + + return; + } + const url = `http://${ip}/api/log`; + RemoteClientLogger.log(`Pushing logs to ${url}`); + const response = await CloudRunnerSystem.Run(`curl -X POST -d "${logs}" ${url}`, false, true); + RemoteClientLogger.log(`Pushed logs to ${url} ${response}`); } async listResources(): Promise { @@ -232,6 +240,7 @@ class Kubernetes implements ProviderInterface { ) { for (let index = 0; index < 3; index++) { try { + const ip = await KubernetesLogService.createLogDeployment(this.namespace, this.kubeClientApps, this.kubeClient); const jobSpec = KubernetesJobSpecFactory.getJobSpec( commands, image, @@ -246,9 +255,12 @@ class Kubernetes implements ProviderInterface { this.jobName, k8s, this.containerName, + ip, ); await new Promise((promise) => setTimeout(promise, 15000)); - await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api); + + // await KubernetesRole.createRole(this.serviceAccountName, this.namespace, this.rbacAuthorizationV1Api); + const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec); CloudRunnerLogger.log(`Build job created`); await new Promise((promise) => setTimeout(promise, 5000)); diff --git a/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts b/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts index af488c5e..01ca5687 100644 --- a/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts +++ b/src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts @@ -20,6 +20,7 @@ class KubernetesJobSpecFactory { jobName: string, k8s: any, containerName: string, + ip: string = '', ) { const job = new k8s.V1Job(); job.apiVersion = 'batch/v1'; @@ -81,6 +82,7 @@ class KubernetesJobSpecFactory { return environmentVariable; }), + { name: 'LOG_SERVICE_IP', value: ip }, ], volumeMounts: [ { diff --git a/src/model/cloud-runner/providers/k8s/kubernetes-log-service.ts b/src/model/cloud-runner/providers/k8s/kubernetes-log-service.ts new file mode 100644 index 00000000..70d418fc --- /dev/null +++ b/src/model/cloud-runner/providers/k8s/kubernetes-log-service.ts @@ -0,0 +1,153 @@ +import { CoreV1Api } from '@kubernetes/client-node'; +import * as k8s from '@kubernetes/client-node'; +class KubernetesLogService { + // static async function, creates a deployment and service + static async createLogService(serviceAccountName: string, namespace: string, kubeClient: CoreV1Api) { + const serviceAccount = new k8s.V1ServiceAccount(); + serviceAccount.apiVersion = 'v1'; + serviceAccount.kind = 'ServiceAccount'; + serviceAccount.metadata = { + name: serviceAccountName, + }; + serviceAccount.automountServiceAccountToken = true; + + return kubeClient.createNamespacedServiceAccount(namespace, serviceAccount); + } + + static async deleteLogService(serviceAccountName: string, namespace: string, kubeClient: CoreV1Api) { + await kubeClient.deleteNamespacedServiceAccount(serviceAccountName, namespace); + } + + static async createLogDeployment(namespace: string, kubeClient: k8s.AppsV1Api, kubeClientCore: CoreV1Api) { + // json + /* + apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + service: http-fileserver + name: http-fileserver +spec: + replicas: 1 + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + service: http-fileserver + spec: + containers: + - image: my-docker-hub-account/http-fileserver-kubernetes:latest + imagePullPolicy: Always + name: http-fileserver + resources: {} + restartPolicy: Always +status: {} + */ + // create a deployment with above json + const deployment = new k8s.V1Deployment(); + deployment.apiVersion = 'apps/v1'; + deployment.kind = 'Deployment'; + deployment.metadata = { + name: 'http-fileserver', + labels: { + service: 'http-fileserver', + }, + }; + deployment.spec = { + selector: { + matchLabels: { + service: 'http-fileserver', + }, + }, + replicas: 1, + strategy: {}, + template: { + metadata: { + labels: { + service: 'http-fileserver', + }, + }, + spec: { + containers: [ + { + image: 'my-docker-hub-account/http-fileserver-kubernetes:latest', + imagePullPolicy: 'Always', + name: 'http-fileserver', + resources: {}, + }, + ], + restartPolicy: 'Always', + }, + }, + }; + await kubeClient.createNamespacedDeployment(namespace, deployment); + await this.createLogServiceExpose(namespace, kubeClientCore); + + // wait in loop until serivce ip address is exposed + + for (let index = 0; index < 10; index++) { + // wait for service to share ip address + await new Promise((resolve) => setTimeout(resolve, 10000)); + + // get ip address of service + const service = await kubeClientCore.readNamespacedService('http-fileserver', namespace); + const ip = service.body.status?.loadBalancer?.ingress?.[0]?.ip; + if (ip) { + return ip; + } + } + } + + // create kubernetes service to expose deployment + static async createLogServiceExpose(namespace: string, kubeClient: CoreV1Api) { + // json + /* + apiVersion: v1 + kind: Service + metadata: + creationTimestamp: null + labels: + service: http-fileserver + name: http-fileserver + spec: + ports: + - name: 80-80 + port: 80 + protocol: TCP + targetPort: 80 + selector: + service: http-fileserver + type: LoadBalancer + status: + loadBalancer: {} + */ + // create a service with above json + const service = new k8s.V1Service(); + service.apiVersion = 'v1'; + service.kind = 'Service'; + service.metadata = { + name: 'http-fileserver', + labels: { + service: 'http-fileserver', + }, + }; + service.spec = { + ports: [ + { + name: '80-80', + port: 80, + protocol: 'TCP', + targetPort: 80, + }, + ], + selector: { + service: 'http-fileserver', + }, + type: 'LoadBalancer', + }; + await kubeClient.createNamespacedService(namespace, service); + } +} +export default KubernetesLogService; diff --git a/src/model/cloud-runner/remote-client/remote-client-logger.ts b/src/model/cloud-runner/remote-client/remote-client-logger.ts index 0085f271..84c1b7f6 100644 --- a/src/model/cloud-runner/remote-client/remote-client-logger.ts +++ b/src/model/cloud-runner/remote-client/remote-client-logger.ts @@ -39,6 +39,7 @@ export class RemoteClientLogger { return; } CloudRunnerLogger.log(`Collected Logs`); + CloudRunnerLogger.log(process.env[`LOG_SERVICE_IP`] || ``); const logs = fs.readFileSync(RemoteClientLogger.LogFilePath).toString(); CloudRunnerLogger.log(logs); await Kubernetes.Instance.PushLogUpdate(logs);