Skip artifact upload if file name already exists.
ClosedAll Users
Actions

Authored by jeschkies on Jun 20 2017, 4:26 PM.

Details

Reviewers

kensipe
timcharper

Commits

rMARATHON0b8688daf43c: Skip artifact upload if file name already exists.
rMARATHONe5192d419fa6: Skip artifact upload if file name already exists.

JIRA Issues

JIRA MARATHON-7783 Marathon build infrastructure: have a unique URL for each build artifact

Summary

We should skip uploads based on file names and not based on the sha1 sum
since this changes for each build.

Test Plan

pipeline

Diff Detail

Repository

rMARATHON marathon

Lint

Automatic diff as part of commit; lint not applicable.

Unit

Automatic diff as part of commit; unit tests not applicable.

jeschkies created this revision.Jun 20 2017, 4:26 PM

Herald added a subscriber: marathon-dev. · View Herald TranscriptJun 20 2017, 4:26 PM

less is more! love when a reduction in code is better!

This revision is now accepted and ready to land.Jun 20 2017, 8:45 PM

timcharper accepted this revision.Jun 21 2017, 1:21 AM

Closed by commit rMARATHONe5192d419fa6: Skip artifact upload if file name already exists. (authored by jeschkies, committed by timcharper). · Explain WhyJun 21 2017, 1:22 AM

This revision was automatically updated to reflect the committed changes.

No. This was broken guys! Jenkins did not accept.

Revision Contents

			Path	Packages
M			ci/awsClient.sc (58 lines)

Diff	ID	Base	Description	Created	Lint	Unit
Base			Base
Diff 1	3569	f364f71		Jun 20 2017, 4:25 PM	★	★
Diff 2	3576	a129f47	rMARATHONe5192d419fa633f7ccd61dc82be78d016706f402	Jun 21 2017, 1:22 AM	★	★

Commit	Tree	Parents	Author	Summary	Date
84edcba5d27a	f9ad401b2aa0	f364f717b13c	Karsten Jeschkies	Skip artifact upload if file name already exists. (Show More…)	Jun 20 2017, 4:23 PM

Diff 3576

View Options

ci/awsClient.sc

Show All 33 Lines
34	34
35	35	/**
36	36	* Returns the default folder for the filekey
37	37	*/
38	38	def getDefaultFileKey(fileName: String): String = {
39	39	return s"${DEFAULT_FOLDER}/${fileName}"
40	40	}
41	41
42		/**
43		* Returns the SHA from the sha file for the file provided.
44		* ex. file == "README.txt", it has a corresponding "README.txt.sha1" as a sha file.
45		* a temp sha file of "README.txt.sha1.tmp" will be created and deleted in the process of getting s3 sha
46		* Returns a "" if sha file doesn't exist.
47		*/
48		def readShaFromS3File(file: Path): Option[String] = {
49		val shaKey: String = getDefaultFileKey(fileUtil.getShaPathForFile(file).last)
50		readFileFromS3(DEFAULT_BUCKET, shaKey)
51		}
52
53		def readFileFromS3(bucket: String, key: String): Option[String] =
54		if(doesS3FileExist(DEFAULT_BUCKET, key)) { fileUtil.withTempFile { tempFile =>
55		// read file to temp
56		downloadFileFromS3(DEFAULT_BUCKET, key, tempFile)
57		Some(read! tempFile)
58		}}
59		else {
60		None
61		}
62
63	42	def skip(file: Path): Unit = println(s"Skipping File: ${file.last} already exists on S3 at ${getDefaultFileKey(file.last)}.")
64	43
65	44	/**
66	45	* Uploads marathon artifacts to the default bucket, using the env var credentials.
67		* Upload process creates the sha1 file
68		* If file is on s3 and the sha1 file has a valid sha, it does NOT upload. (these files are big)
	46	* Upload process creates the sha1 file.
	47	* If file name is on s3, it does NOT upload (these files are big). We cannot
	48	* compare the sha1 sums because they change for each build of the same commit.
	49	* However, our artifact names are unique for each commit.
69	50	*/
70	51	def archiveArtifact(uploadFile: Path): Unit = {
71		// is already uploaded and the sha file is the correct sha
72		if(alreadyUploaded(uploadFile)) skip(uploadFile)
	52	// is already uploaded.
	53	if(doesS3FileExist(uploadFile)) skip(uploadFile)
73	54	else uploadFileAndSha(uploadFile)
74	55	}
75	56
76	57	/**
77	58	* Uploads marathon artifacts to the default bucket, using the env var credentials.
78	59	* Upload process creates the sha1 file
79	60	*/
80	61	def uploadFileAndSha(uploadFile: Path): Unit = {
81	62
82	63	val fileKey: String = getDefaultFileKey(uploadFile.last)
83	64	val shaFile = fileUtil.writeSha1ForFile(uploadFile)
84	65	val shaFileKey: String = getDefaultFileKey(shaFile.last)
85	66
86	67	println(s"${fileKey} uploading to S3")
87	68	uploadFileToS3(DEFAULT_BUCKET, fileKey, uploadFile)
88	69
89	70	println(s"${shaFileKey} uploading to S3")
90	71	uploadFileToS3(DEFAULT_BUCKET, shaFileKey, shaFile)
91		}
92
93		/**
94		* returns True if the s3 sha file has a sha == to local sha and the
95		* artifact file is already uploaded on s3.
96		*/
97		def alreadyUploaded(uploadFile: Path): Boolean = {
98		val fileKey: String = getDefaultFileKey(uploadFile.last)
99		val localSha = read! fileUtil.writeSha1ForFile(uploadFile)
100		val s3Sha = readShaFromS3File(uploadFile)
101		s3Sha.fold(false)(_ == localSha) && doesS3FileExist(DEFAULT_BUCKET, fileKey)
102		}
103
104		/**
105		* Downloads a file from the S3 bucket
106		*/
107		def downloadFileFromS3(bucket: String, fileName: String, downloadFile: Path): Unit = {
108		val transfer: TransferManager = TransferManagerBuilder.standard().withS3Client(createS3Client()).build()
109		val download: Download = transfer.download(bucket, fileName, downloadFile.toIO)
110		while(!download.isDone()) {
111		val progress = download.getProgress()
112		println(s"Downloading (${fileName}): ${progress.getPercentTransferred()} % ${download.getState()}")
113		Thread.sleep(1000)
114		}
115
116		transfer.shutdownNow(true)
117		assert(download.getState() == TransferState.Completed, s"Download finished with ${download.getState()}")
118	72	}
119	73
120	74
121	75	/**
122	76	* Uploads a file to the S3 bucket
123	77	*/
124	78	def uploadFileToS3(bucket: String, fileName: String, uploadFile: Path): Unit = {
125	79	val transfer: TransferManager = TransferManagerBuilder.standard().withS3Client(createS3Client()).build()
Show All 13 Lines