mirror of
				https://github.com/actions/cache.git
				synced 2025-10-31 18:34:19 +08:00 
			
		
		
		
	Format and update tests
This commit is contained in:
		
							parent
							
								
									2ce22df8c4
								
							
						
					
					
						commit
						cdec5dec0d
					
				| @ -212,14 +212,14 @@ test("save with large cache outputs warning", async () => { | |||||||
|     const IS_WINDOWS = process.platform === "win32"; |     const IS_WINDOWS = process.platform === "win32"; | ||||||
|     const args = IS_WINDOWS |     const args = IS_WINDOWS | ||||||
|         ? [ |         ? [ | ||||||
|               "-cz", |             "-cz", | ||||||
|               "--force-local", |             "--force-local", | ||||||
|               "-f", |             "-f", | ||||||
|               archivePath.replace(/\\/g, "/"), |             archivePath.replace(/\\/g, "/"), | ||||||
|               "-C", |             "-C", | ||||||
|               cachePath.replace(/\\/g, "/"), |             cachePath.replace(/\\/g, "/"), | ||||||
|               "." |             "." | ||||||
|           ] |         ] | ||||||
|         : ["-cz", "-f", archivePath, "-C", cachePath, "."]; |         : ["-cz", "-f", archivePath, "-C", cachePath, "."]; | ||||||
| 
 | 
 | ||||||
|     expect(execMock).toHaveBeenCalledTimes(1); |     expect(execMock).toHaveBeenCalledTimes(1); | ||||||
| @ -259,6 +259,11 @@ test("save with server error outputs warning", async () => { | |||||||
|     const cachePath = path.resolve(inputPath); |     const cachePath = path.resolve(inputPath); | ||||||
|     testUtils.setInput(Inputs.Path, inputPath); |     testUtils.setInput(Inputs.Path, inputPath); | ||||||
| 
 | 
 | ||||||
|  |     const cacheId = 4; | ||||||
|  |     const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => { | ||||||
|  |         return Promise.resolve(cacheId); | ||||||
|  |     }); | ||||||
|  | 
 | ||||||
|     const execMock = jest.spyOn(exec, "exec"); |     const execMock = jest.spyOn(exec, "exec"); | ||||||
| 
 | 
 | ||||||
|     const saveCacheMock = jest |     const saveCacheMock = jest | ||||||
| @ -269,26 +274,29 @@ test("save with server error outputs warning", async () => { | |||||||
| 
 | 
 | ||||||
|     await run(); |     await run(); | ||||||
| 
 | 
 | ||||||
|  |     expect(reserveCacheMock).toHaveBeenCalledTimes(1); | ||||||
|  |     expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey); | ||||||
|  | 
 | ||||||
|     const archivePath = path.join("/foo/bar", "cache.tgz"); |     const archivePath = path.join("/foo/bar", "cache.tgz"); | ||||||
| 
 | 
 | ||||||
|     const IS_WINDOWS = process.platform === "win32"; |     const IS_WINDOWS = process.platform === "win32"; | ||||||
|     const args = IS_WINDOWS |     const args = IS_WINDOWS | ||||||
|         ? [ |         ? [ | ||||||
|               "-cz", |             "-cz", | ||||||
|               "--force-local", |             "--force-local", | ||||||
|               "-f", |             "-f", | ||||||
|               archivePath.replace(/\\/g, "/"), |             archivePath.replace(/\\/g, "/"), | ||||||
|               "-C", |             "-C", | ||||||
|               cachePath.replace(/\\/g, "/"), |             cachePath.replace(/\\/g, "/"), | ||||||
|               "." |             "." | ||||||
|           ] |         ] | ||||||
|         : ["-cz", "-f", archivePath, "-C", cachePath, "."]; |         : ["-cz", "-f", archivePath, "-C", cachePath, "."]; | ||||||
| 
 | 
 | ||||||
|     expect(execMock).toHaveBeenCalledTimes(1); |     expect(execMock).toHaveBeenCalledTimes(1); | ||||||
|     expect(execMock).toHaveBeenCalledWith(`"tar"`, args); |     expect(execMock).toHaveBeenCalledWith(`"tar"`, args); | ||||||
| 
 | 
 | ||||||
|     expect(saveCacheMock).toHaveBeenCalledTimes(1); |     expect(saveCacheMock).toHaveBeenCalledTimes(1); | ||||||
|     expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); |     expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath); | ||||||
| 
 | 
 | ||||||
|     expect(logWarningMock).toHaveBeenCalledTimes(1); |     expect(logWarningMock).toHaveBeenCalledTimes(1); | ||||||
|     expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred"); |     expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred"); | ||||||
| @ -321,32 +329,40 @@ test("save with valid inputs uploads a cache", async () => { | |||||||
|     const cachePath = path.resolve(inputPath); |     const cachePath = path.resolve(inputPath); | ||||||
|     testUtils.setInput(Inputs.Path, inputPath); |     testUtils.setInput(Inputs.Path, inputPath); | ||||||
| 
 | 
 | ||||||
|  |     const cacheId = 4; | ||||||
|  |     const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => { | ||||||
|  |         return Promise.resolve(cacheId); | ||||||
|  |     }); | ||||||
|  | 
 | ||||||
|     const execMock = jest.spyOn(exec, "exec"); |     const execMock = jest.spyOn(exec, "exec"); | ||||||
| 
 | 
 | ||||||
|     const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache"); |     const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache"); | ||||||
| 
 | 
 | ||||||
|     await run(); |     await run(); | ||||||
| 
 | 
 | ||||||
|  |     expect(reserveCacheMock).toHaveBeenCalledTimes(1); | ||||||
|  |     expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey); | ||||||
|  | 
 | ||||||
|     const archivePath = path.join("/foo/bar", "cache.tgz"); |     const archivePath = path.join("/foo/bar", "cache.tgz"); | ||||||
| 
 | 
 | ||||||
|     const IS_WINDOWS = process.platform === "win32"; |     const IS_WINDOWS = process.platform === "win32"; | ||||||
|     const args = IS_WINDOWS |     const args = IS_WINDOWS | ||||||
|         ? [ |         ? [ | ||||||
|               "-cz", |             "-cz", | ||||||
|               "--force-local", |             "--force-local", | ||||||
|               "-f", |             "-f", | ||||||
|               archivePath.replace(/\\/g, "/"), |             archivePath.replace(/\\/g, "/"), | ||||||
|               "-C", |             "-C", | ||||||
|               cachePath.replace(/\\/g, "/"), |             cachePath.replace(/\\/g, "/"), | ||||||
|               "." |             "." | ||||||
|           ] |         ] | ||||||
|         : ["-cz", "-f", archivePath, "-C", cachePath, "."]; |         : ["-cz", "-f", archivePath, "-C", cachePath, "."]; | ||||||
| 
 | 
 | ||||||
|     expect(execMock).toHaveBeenCalledTimes(1); |     expect(execMock).toHaveBeenCalledTimes(1); | ||||||
|     expect(execMock).toHaveBeenCalledWith(`"tar"`, args); |     expect(execMock).toHaveBeenCalledWith(`"tar"`, args); | ||||||
| 
 | 
 | ||||||
|     expect(saveCacheMock).toHaveBeenCalledTimes(1); |     expect(saveCacheMock).toHaveBeenCalledTimes(1); | ||||||
|     expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath); |     expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath); | ||||||
| 
 | 
 | ||||||
|     expect(failedMock).toHaveBeenCalledTimes(0); |     expect(failedMock).toHaveBeenCalledTimes(0); | ||||||
| }); | }); | ||||||
|  | |||||||
| @ -107,9 +107,7 @@ export async function downloadCache( | |||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // Reserve Cache
 | // Reserve Cache
 | ||||||
| export async function reserveCache( | export async function reserveCache(key: string): Promise<number> { | ||||||
|     key: string |  | ||||||
| ): Promise<number> { |  | ||||||
|     const restClient = createRestClient(); |     const restClient = createRestClient(); | ||||||
| 
 | 
 | ||||||
|     const reserveCacheRequest: ReserveCacheRequest = { |     const reserveCacheRequest: ReserveCacheRequest = { | ||||||
| @ -133,14 +131,6 @@ function getContentRange(start: number, end: number): string { | |||||||
|     return `bytes ${start}-${end}/*`; |     return `bytes ${start}-${end}/*`; | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| // function bufferToStream(buffer: Buffer): NodeJS.ReadableStream {
 |  | ||||||
| //     const stream = new Duplex();
 |  | ||||||
| //     stream.push(buffer);
 |  | ||||||
| //     stream.push(null);
 |  | ||||||
| 
 |  | ||||||
| //     return stream;
 |  | ||||||
| // }
 |  | ||||||
| 
 |  | ||||||
| async function uploadChunk( | async function uploadChunk( | ||||||
|     restClient: RestClient, |     restClient: RestClient, | ||||||
|     resourceUrl: string, |     resourceUrl: string, | ||||||
| @ -148,14 +138,87 @@ async function uploadChunk( | |||||||
|     start: number, |     start: number, | ||||||
|     end: number |     end: number | ||||||
| ): Promise<IRestResponse<void>> { | ): Promise<IRestResponse<void>> { | ||||||
|     core.debug(`Uploading chunk of size ${end - start + 1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`); |     core.debug( | ||||||
|  |         `Uploading chunk of size ${end - | ||||||
|  |             start + | ||||||
|  |             1} bytes at offset ${start} with content range: ${getContentRange( | ||||||
|  |             start, | ||||||
|  |             end | ||||||
|  |         )}` | ||||||
|  |     ); | ||||||
|     const requestOptions = getRequestOptions(); |     const requestOptions = getRequestOptions(); | ||||||
|     requestOptions.additionalHeaders = { |     requestOptions.additionalHeaders = { | ||||||
|         "Content-Type": "application/octet-stream", |         "Content-Type": "application/octet-stream", | ||||||
|         "Content-Range": getContentRange(start, end) |         "Content-Range": getContentRange(start, end) | ||||||
|     }; |     }; | ||||||
| 
 | 
 | ||||||
|     return await restClient.uploadStream<void>("PATCH", resourceUrl, data, requestOptions); |     return await restClient.uploadStream<void>( | ||||||
|  |         "PATCH", | ||||||
|  |         resourceUrl, | ||||||
|  |         data, | ||||||
|  |         requestOptions | ||||||
|  |     ); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | async function uploadFile( | ||||||
|  |     restClient: RestClient, | ||||||
|  |     cacheId: number, | ||||||
|  |     archivePath: string | ||||||
|  | ): Promise<void> { | ||||||
|  |     // Upload Chunks
 | ||||||
|  |     const fileSize = fs.statSync(archivePath).size; | ||||||
|  |     const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString(); | ||||||
|  |     const responses: IRestResponse<void>[] = []; | ||||||
|  |     const fd = fs.openSync(archivePath, "r"); | ||||||
|  | 
 | ||||||
|  |     const concurrency = 4; // # of HTTP requests in parallel
 | ||||||
|  |     const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks
 | ||||||
|  |     core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`); | ||||||
|  | 
 | ||||||
|  |     const parallelUploads = [...new Array(concurrency).keys()]; | ||||||
|  |     core.debug("Awaiting all uploads"); | ||||||
|  |     let offset = 0; | ||||||
|  |     await Promise.all( | ||||||
|  |         parallelUploads.map(async () => { | ||||||
|  |             while (offset < fileSize) { | ||||||
|  |                 const chunkSize = | ||||||
|  |                     offset + MAX_CHUNK_SIZE > fileSize | ||||||
|  |                         ? fileSize - offset | ||||||
|  |                         : MAX_CHUNK_SIZE; | ||||||
|  |                 const start = offset; | ||||||
|  |                 const end = offset + chunkSize - 1; | ||||||
|  |                 offset += MAX_CHUNK_SIZE; | ||||||
|  |                 const chunk = fs.createReadStream(archivePath, { | ||||||
|  |                     fd, | ||||||
|  |                     start, | ||||||
|  |                     end, | ||||||
|  |                     autoClose: false | ||||||
|  |                 }); | ||||||
|  |                 responses.push( | ||||||
|  |                     await uploadChunk( | ||||||
|  |                         restClient, | ||||||
|  |                         resourceUrl, | ||||||
|  |                         chunk, | ||||||
|  |                         start, | ||||||
|  |                         end | ||||||
|  |                     ) | ||||||
|  |                 ); | ||||||
|  |             } | ||||||
|  |         }) | ||||||
|  |     ); | ||||||
|  | 
 | ||||||
|  |     fs.closeSync(fd); | ||||||
|  | 
 | ||||||
|  |     const failedResponse = responses.find( | ||||||
|  |         x => !isSuccessStatusCode(x.statusCode) | ||||||
|  |     ); | ||||||
|  |     if (failedResponse) { | ||||||
|  |         throw new Error( | ||||||
|  |             `Cache service responded with ${failedResponse.statusCode} during chunk upload.` | ||||||
|  |         ); | ||||||
|  |     } | ||||||
|  | 
 | ||||||
|  |     return; | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| async function commitCache( | async function commitCache( | ||||||
| @ -172,44 +235,6 @@ async function commitCache( | |||||||
|     ); |     ); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| async function uploadFile(restClient: RestClient, cacheId: number, archivePath: string): Promise<void> { |  | ||||||
|     // Upload Chunks
 |  | ||||||
|     const fileSize = fs.statSync(archivePath).size; |  | ||||||
|     const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString(); |  | ||||||
|     const responses: IRestResponse<void>[] = []; |  | ||||||
|     const fd = fs.openSync(archivePath, "r"); |  | ||||||
| 
 |  | ||||||
|     const concurrency = 16; // # of HTTP requests in parallel
 |  | ||||||
|     const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks
 |  | ||||||
|     core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`); |  | ||||||
|     const parallelUploads = [...new Array(concurrency).keys()]; |  | ||||||
|     core.debug("Awaiting all uploads"); |  | ||||||
|     let offset = 0; |  | ||||||
|     await Promise.all(parallelUploads.map(async () => { |  | ||||||
|         while (offset < fileSize) { |  | ||||||
|             const chunkSize = offset + MAX_CHUNK_SIZE > fileSize ? fileSize - offset : MAX_CHUNK_SIZE; |  | ||||||
|             const start = offset; |  | ||||||
|             const end = offset + chunkSize - 1; |  | ||||||
|             offset += MAX_CHUNK_SIZE; |  | ||||||
|             const chunk = fs.createReadStream(archivePath, { fd, start, end, autoClose: false }); |  | ||||||
|             responses.push(await uploadChunk(restClient, resourceUrl, chunk, start, end)); |  | ||||||
|         } |  | ||||||
|     })); |  | ||||||
| 
 |  | ||||||
|     fs.closeSync(fd); |  | ||||||
| 
 |  | ||||||
|     const failedResponse = responses.find( |  | ||||||
|         x => !isSuccessStatusCode(x.statusCode) |  | ||||||
|     ); |  | ||||||
|     if (failedResponse) { |  | ||||||
|         throw new Error( |  | ||||||
|             `Cache service responded with ${failedResponse.statusCode} during chunk upload.` |  | ||||||
|         ); |  | ||||||
|     } |  | ||||||
| 
 |  | ||||||
|     return; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| export async function saveCache( | export async function saveCache( | ||||||
|     cacheId: number, |     cacheId: number, | ||||||
|     archivePath: string |     archivePath: string | ||||||
| @ -219,8 +244,8 @@ export async function saveCache( | |||||||
|     core.debug("Upload cache"); |     core.debug("Upload cache"); | ||||||
|     await uploadFile(restClient, cacheId, archivePath); |     await uploadFile(restClient, cacheId, archivePath); | ||||||
| 
 | 
 | ||||||
|     core.debug("Commiting cache"); |  | ||||||
|     // Commit Cache
 |     // Commit Cache
 | ||||||
|  |     core.debug("Commiting cache"); | ||||||
|     const cacheSize = utils.getArchiveFileSize(archivePath); |     const cacheSize = utils.getArchiveFileSize(archivePath); | ||||||
|     const commitCacheResponse = await commitCache( |     const commitCacheResponse = await commitCache( | ||||||
|         restClient, |         restClient, | ||||||
|  | |||||||
| @ -82,7 +82,6 @@ async function run(): Promise<void> { | |||||||
|                 cacheEntry?.archiveLocation, |                 cacheEntry?.archiveLocation, | ||||||
|                 archivePath |                 archivePath | ||||||
|             ); |             ); | ||||||
|             await exec(`md5sum`, [archivePath]); |  | ||||||
| 
 | 
 | ||||||
|             const archiveFileSize = utils.getArchiveFileSize(archivePath); |             const archiveFileSize = utils.getArchiveFileSize(archivePath); | ||||||
|             core.info( |             core.info( | ||||||
|  | |||||||
| @ -38,7 +38,9 @@ async function run(): Promise<void> { | |||||||
|         core.debug("Reserving Cache"); |         core.debug("Reserving Cache"); | ||||||
|         const cacheId = await cacheHttpClient.reserveCache(primaryKey); |         const cacheId = await cacheHttpClient.reserveCache(primaryKey); | ||||||
|         if (cacheId < 0) { |         if (cacheId < 0) { | ||||||
|             core.info(`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`); |             core.info( | ||||||
|  |                 `Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.` | ||||||
|  |             ); | ||||||
|             return; |             return; | ||||||
|         } |         } | ||||||
|         core.debug(`Cache ID: ${cacheId}`); |         core.debug(`Cache ID: ${cacheId}`); | ||||||
| @ -84,7 +86,6 @@ async function run(): Promise<void> { | |||||||
|             return; |             return; | ||||||
|         } |         } | ||||||
| 
 | 
 | ||||||
|         await exec(`md5sum`, [archivePath]); |  | ||||||
|         core.debug("Saving Cache"); |         core.debug("Saving Cache"); | ||||||
|         await cacheHttpClient.saveCache(cacheId, archivePath); |         await cacheHttpClient.saveCache(cacheId, archivePath); | ||||||
|     } catch (error) { |     } catch (error) { | ||||||
|  | |||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user
	 Josh Gross
						Josh Gross