7 7 | import aws.sdk.kotlin.services.s3.*
|
8 8 | import aws.sdk.kotlin.services.s3.model.*
|
9 9 | import aws.sdk.kotlin.testing.PRINTABLE_CHARS
|
10 10 | import aws.sdk.kotlin.testing.withAllEngines
|
11 11 | import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext
|
12 12 | import aws.smithy.kotlin.runtime.content.*
|
13 13 | import aws.smithy.kotlin.runtime.hashing.sha256
|
14 14 | import aws.smithy.kotlin.runtime.http.HttpException
|
15 15 | import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor
|
16 16 | import aws.smithy.kotlin.runtime.http.request.HttpRequest
|
17 + | import aws.smithy.kotlin.runtime.io.use
|
18 + | import aws.smithy.kotlin.runtime.testing.AfterAll
|
19 + | import aws.smithy.kotlin.runtime.testing.BeforeAll
|
17 20 | import aws.smithy.kotlin.runtime.testing.RandomTempFile
|
18 21 | import aws.smithy.kotlin.runtime.text.encoding.encodeToHex
|
19 22 | import kotlinx.coroutines.async
|
20 23 | import kotlinx.coroutines.awaitAll
|
21 24 | import kotlinx.coroutines.flow.flow
|
22 - | import kotlinx.coroutines.flow.toList
|
23 25 | import kotlinx.coroutines.runBlocking
|
24 - | import kotlinx.coroutines.withTimeout
|
25 - | import org.junit.jupiter.api.AfterAll
|
26 - | import org.junit.jupiter.api.BeforeAll
|
27 - | import org.junit.jupiter.api.TestInstance
|
28 - | import java.io.File
|
29 - | import java.util.*
|
26 + | import kotlin.jvm.JvmStatic
|
27 + | import kotlin.random.Random
|
30 28 | import kotlin.test.*
|
31 - | import kotlin.time.Duration.Companion.seconds
|
32 29 |
|
33 - | /**
|
34 - | * Tests for bucket operations and presigner
|
35 - | */
|
36 - | @TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
37 30 | class S3BucketOpsIntegrationTest {
|
38 - | private val client = S3Client {
|
39 - | region = S3TestUtils.DEFAULT_REGION
|
40 - | }
|
41 - |
|
42 - | private lateinit var testBucket: String
|
43 - |
|
44 - | @BeforeAll
|
45 - | fun createResources(): Unit = runBlocking {
|
46 - | testBucket = S3TestUtils.getTestBucket(client)
|
47 - | }
|
31 + | companion object {
|
32 + | private lateinit var client: S3Client
|
33 + | private lateinit var testBucket: String
|
34 + |
|
35 + | @BeforeAll
|
36 + | @JvmStatic
|
37 + | fun setup() = runBlocking {
|
38 + | client = S3Client {
|
39 + | region = S3TestUtils.DEFAULT_REGION
|
40 + | }
|
41 + | testBucket = S3TestUtils.getOrCreateSharedBucket(client)
|
42 + | }
|
48 43 |
|
49 - | @AfterAll
|
50 - | fun cleanup() = runBlocking {
|
51 - | S3TestUtils.deleteBucketAndAllContents(client, testBucket)
|
52 - | client.close()
|
44 + | @AfterAll
|
45 + | @JvmStatic
|
46 + | fun cleanup(): Unit = runBlocking {
|
47 + | S3TestUtils.cleanupSharedBucket(client)
|
48 + | client.close()
|
49 + | }
|
53 50 | }
|
54 51 |
|
55 52 | @Test
|
56 - | fun testPutObjectFromMemory(): Unit = runBlocking {
|
53 + | fun testPutObjectFromMemory() = runBlocking {
|
57 54 | val contents = """
|
58 55 | A lep is a ball.
|
59 56 | A tay is a hammer.
|
60 57 | A korf is a tiger.
|
61 58 | A flix is a comb.
|
62 59 | A wogsin is a gift.
|
63 60 | """.trimIndent()
|
64 61 |
|
65 62 | val keyName = "put-obj-from-memory.txt"
|
66 63 |
|
67 64 | client.putObject {
|
68 65 | bucket = testBucket
|
69 66 | key = keyName
|
70 67 | body = ByteStream.fromString(contents)
|
71 68 | }
|
72 69 |
|
73 70 | val req = GetObjectRequest {
|
74 71 | bucket = testBucket
|
75 72 | key = keyName
|
76 73 | }
|
77 74 | val roundTrippedContents = client.getObject(req) { it.body?.decodeToString() }
|
78 75 |
|
79 76 | assertEquals(contents, roundTrippedContents)
|
80 77 | }
|
81 78 |
|
82 79 | @Test
|
83 - | fun testPutObjectFromFile(): Unit = runBlocking {
|
84 - | val tempFile = RandomTempFile(1024)
|
85 - | val keyName = "put-obj-from-file.txt"
|
86 - |
|
87 - | // This test fails sporadically (by never completing)
|
88 - | // see https://github.com/awslabs/aws-sdk-kotlin/issues/282
|
89 - | withTimeout(5.seconds) {
|
90 - | client.putObject {
|
91 - | bucket = testBucket
|
92 - | key = keyName
|
93 - | body = ByteStream.fromFile(tempFile)
|
94 - | }
|
95 - | }
|
96 - |
|
97 - | val req = GetObjectRequest {
|
98 - | bucket = testBucket
|
99 - | key = keyName
|
100 - | }
|
101 - | val roundTrippedContents = client.getObject(req) { it.body?.decodeToString() }
|
102 - |
|
103 - | val contents = tempFile.readText()
|
104 - | assertEquals(contents, roundTrippedContents)
|
105 - | }
|
106 - |
|
107 - | @Test
|
108 - | fun testPutObjectWithToByteStreamAndContentLength(): Unit = runBlocking {
|
80 + | fun testPutObjectWithToByteStreamAndContentLength() = runBlocking<Unit> {
|
109 81 | // See https://github.com/awslabs/aws-sdk-kotlin/issues/1249
|
110 82 | val keyName = "toByteStream-contentLength.txt"
|
111 83 | val arr = "Hello!".encodeToByteArray()
|
112 84 |
|
113 85 | client.putObject {
|
114 86 | bucket = testBucket
|
115 87 | key = keyName
|
116 88 | body = flow { emit(arr) }.toByteStream(this@runBlocking, arr.size.toLong())
|
117 89 | contentLength = arr.size.toLong()
|
118 90 | }
|
119 91 | }
|
120 92 |
|
121 93 | @Test
|
122 - | fun testGetEmptyObject(): Unit = runBlocking {
|
94 + | fun testGetEmptyObject() = runBlocking {
|
123 95 | // See https://github.com/awslabs/aws-sdk-kotlin/issues/1014
|
124 96 | val keyName = "get-empty-obj.txt"
|
125 97 |
|
126 98 | client.putObject {
|
127 99 | bucket = testBucket
|
128 100 | key = keyName
|
129 101 | body = ByteStream.fromBytes(byteArrayOf())
|
130 102 | }
|
131 103 |
|
132 104 | val req = GetObjectRequest {
|
133 105 | bucket = testBucket
|
134 106 | key = keyName
|
135 107 | }
|
136 108 | val actualLength = client.getObject(req) { it.contentLength }
|
137 109 | assertEquals(0, actualLength)
|
138 110 | }
|
139 111 |
|
140 112 | @Test
|
141 - | fun testQueryParameterEncoding(): Unit = runBlocking {
|
113 + | fun testQueryParameterEncoding() = runBlocking {
|
142 114 | // see: https://github.com/awslabs/aws-sdk-kotlin/issues/448
|
143 115 |
|
144 116 | // this is mostly a stress test of signing w.r.t query parameter encoding (since
|
145 117 | // delimiter is bound via @httpQuery) and the ability of an HTTP engine to keep
|
146 118 | // the same encoding going out on the wire (e.g. not double percent encoding)
|
147 119 |
|
148 120 | s3WithAllEngines { s3 ->
|
149 121 | s3.listObjects {
|
150 122 | bucket = testBucket
|
151 123 | delimiter = PRINTABLE_CHARS
|
152 124 | prefix = null
|
153 125 | }
|
154 - | // only care that request is accepted, not the results
|
155 126 | }
|
156 127 | }
|
157 128 |
|
158 129 | @Test
|
159 - | fun testPathEncoding(): Unit = runBlocking {
|
130 + | fun testPathEncoding() = runBlocking {
|
160 131 | // this is mostly a stress test of signing w.r.t path encoding (since key is bound
|
161 132 | // via @httpLabel) and the ability of an HTTP engine to keep the same encoding going
|
162 133 | // out on the wire (e.g. not double percent encoding)
|
163 134 |
|
164 135 | // NOTE: S3 provides guidance on choosing object key names: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
|
165 136 | // This test includes all printable chars (including ones S3 recommends avoiding). Users should
|
166 137 | // strive to fall within the guidelines given by S3 though
|
167 138 |
|
168 139 | s3WithAllEngines { s3 ->
|
169 140 | val objKey = "foo$PRINTABLE_CHARS"
|
170 141 | val content = "hello rfc3986"
|
171 142 |
|
172 143 | s3.putObject {
|
173 144 | bucket = testBucket
|
174 145 | key = objKey
|
175 146 | body = ByteStream.fromString(content)
|
176 147 | }
|
177 148 |
|
178 149 | val req = GetObjectRequest {
|
179 150 | bucket = testBucket
|
180 151 | key = objKey
|
181 152 | }
|
182 153 |
|
183 154 | s3.getObject(req) { resp ->
|
184 155 | val actual = resp.body!!.decodeToString()
|
185 156 | assertEquals(content, actual)
|
186 157 | }
|
187 158 | }
|
188 159 | }
|
189 160 |
|
190 161 | @Test
|
191 - | fun testMultipartUpload(): Unit = runBlocking {
|
162 + | fun testMultipartUpload() = runBlocking {
|
192 163 | s3WithAllEngines { s3 ->
|
193 - | val objKey = "test-multipart-${UUID.randomUUID()}"
|
194 - | val contentSize: Long = 8 * 1024 * 1024 // 2 parts
|
164 + | val objKey = "test-multipart-${Random.nextInt()}"
|
165 + | val contentSize: Long = 8 * 1024 * 1024
|
195 166 | val file = RandomTempFile(sizeInBytes = contentSize)
|
196 - | val partSize = 5 * 1024 * 1024 // 5 MB - min part size
|
167 + | val partSize = 5 * 1024 * 1024
|
197 168 |
|
198 169 | val expectedSha256 = file.readBytes().sha256().encodeToHex()
|
199 170 |
|
200 171 | val resp = s3.createMultipartUpload {
|
201 172 | bucket = testBucket
|
202 173 | key = objKey
|
203 174 | }
|
204 175 |
|
205 - | val completedParts = file.chunk(partSize)
|
176 + | val fileBytes = file.readBytes()
|
177 + | val completedParts = fileBytes.chunk(partSize)
|
206 178 | .mapIndexed { idx, chunk ->
|
207 179 | async {
|
208 180 | val uploadResp = s3.uploadPart {
|
209 181 | bucket = testBucket
|
210 182 | key = objKey
|
211 183 | uploadId = resp.uploadId
|
212 - | body = file.asByteStream(chunk)
|
184 + | body = ByteStream.fromBytes(chunk)
|
213 185 | partNumber = idx + 1
|
214 186 | }
|
215 187 |
|
216 188 | CompletedPart {
|
217 189 | partNumber = idx + 1
|
218 190 | eTag = uploadResp.eTag
|
219 191 | }
|
220 192 | }
|
221 193 | }
|
222 194 | .toList()
|
223 195 | .awaitAll()
|
224 196 |
|
225 197 | s3.completeMultipartUpload {
|
226 198 | bucket = testBucket
|
227 199 | key = objKey
|
228 200 | uploadId = resp.uploadId
|
229 201 | multipartUpload {
|
230 202 | parts = completedParts
|
231 203 | }
|
232 204 | }
|
233 205 |
|
234 - | // TOOD - eventually make use of s3 checksums
|
235 206 | val getRequest = GetObjectRequest {
|
236 207 | bucket = testBucket
|
237 208 | key = objKey
|
238 209 | }
|
239 210 | val actualSha256 = s3.getObject(getRequest) { resp ->
|
240 211 | resp.body!!.toByteArray().sha256().encodeToHex()
|
241 212 | }
|
242 213 |
|
243 214 | assertEquals(expectedSha256, actualSha256)
|
215 + | file.delete()
|
244 216 | }
|
245 217 | }
|
246 218 |
|
247 219 | @Test
|
248 - | fun testPutObjectWithChecksum(): Unit = runBlocking {
|
220 + | fun testPutObjectWithChecksum() = runBlocking {
|
249 221 | val contents = "AAAAAAAAAA"
|
250 222 | val keyName = "put-obj-with-checksum.txt"
|
251 223 |
|
252 224 | val resp = client.putObject {
|
253 225 | bucket = testBucket
|
254 226 | key = keyName
|
255 227 | body = ByteStream.fromString(contents)
|
256 228 | checksumAlgorithm = ChecksumAlgorithm.Sha256
|
257 229 | }
|
258 230 |
|
259 231 | val req = GetObjectRequest {
|
260 232 | bucket = testBucket
|
261 233 | key = keyName
|
262 234 | checksumMode = ChecksumMode.Enabled
|
263 235 | }
|
264 236 |
|
265 237 | val roundTrippedContents = client.getObject(req) {
|
266 238 | assertEquals(resp.checksumSha256, it.checksumSha256)
|
267 239 | it.body?.decodeToString()
|
268 240 | }
|
269 241 |
|
270 242 | assertEquals(contents, roundTrippedContents)
|
271 243 | }
|
272 244 |
|
273 245 | @Test
|
274 246 | fun testPutObjectWithIncorrectChecksum(): Unit = runBlocking {
|
275 247 | val contents = "AAAAAAAAAA"
|
276 - |
|
277 248 | val keyName = "put-obj-with-checksum.txt"
|
278 249 |
|
279 250 | val ex = assertFails {
|
280 251 | client.putObject {
|
281 252 | bucket = testBucket
|
282 253 | key = keyName
|
283 254 | body = ByteStream.fromString(contents)
|
284 255 | checksumAlgorithm = ChecksumAlgorithm.Sha256
|
285 256 | checksumSha256 = "blerg"
|
286 257 | }
|
287 258 | }
|
288 259 | ex.message?.let {
|
289 - | assert(it.contains("Value for x-amz-checksum-sha256 header is invalid."))
|
260 + | assertTrue(it.contains("Value for x-amz-checksum-sha256 header is invalid."))
|
290 261 | }
|
291 262 | }
|
292 263 |
|
293 264 | @Test
|
294 265 | fun testWriteGetObjectResponse(): Unit = runBlocking {
|
295 - | // Interceptor which validates the `Host` header against an `expectedHost`
|
296 266 | class WriteGetObjectResponseHostInterceptor(val expectedHost: String) : HttpInterceptor {
|
297 267 | override fun readAfterSigning(context: ProtocolRequestInterceptorContext<Any, HttpRequest>) {
|
298 268 | val req = context.protocolRequest
|
299 269 | assertEquals(expectedHost, req.headers["Host"])
|
300 270 | }
|
301 271 | }
|
302 272 |
|
303 273 | val expectedHost = "s3-object-lambda.${client.config.region}.amazonaws.com"
|
304 274 |
|
305 275 | client.withConfig {
|
306 276 | interceptors = mutableListOf(WriteGetObjectResponseHostInterceptor(expectedHost))
|
307 277 | }.use {
|
308 278 | // The request is expected to fail because we don't have the proper infrastructure set up for the request
|
309 279 | // (S3 Access Point, Lambda Function, etc.)
|
310 280 | val ex = assertFailsWith<HttpException> {
|
311 281 | it.writeGetObjectResponse {}
|
312 282 | }
|
313 - | assertContains(ex.message!!, "$expectedHost")
|
283 + | // JVM error message contains the host, Native error message is generic DNS error
|
284 + | assertTrue(
|
285 + | ex.message!!.contains(expectedHost) || ex.message!!.contains("Host name was invalid for dns resolution"),
|
286 + | "Expected error message to contain either '$expectedHost' or 'Host name was invalid for dns resolution', but got: ${ex.message}",
|
287 + | )
|
314 288 | }
|
315 289 | }
|
316 290 |
|
317 291 | @Test
|
318 - | fun testHeadObjectForbidden(): Unit = runBlocking {
|
292 + | fun testHeadObjectForbidden() = runBlocking {
|
319 293 | val ex = assertFailsWith<S3Exception> {
|
320 294 | client.withConfig {
|
321 295 | region = "us-east-1"
|
322 296 | }.headObject {
|
323 297 | bucket = "bucket"
|
324 298 | key = "any-key.txt"
|
325 299 | }
|
326 300 | }
|
327 301 |
|
328 302 | assertContains(ex.message, "Service returned error code 403: Forbidden")
|
329 303 | assertEquals("403: Forbidden", ex.sdkErrorMetadata.errorCode!!)
|
330 304 | }
|
331 305 | }
|
332 306 |
|
333 - | // generate sequence of "chunks" where each range defines the inclusive start and end bytes
|
334 - | internal fun File.chunk(partSize: Int): Sequence<LongRange> = (0 until length() step partSize.toLong()).asSequence().map {
|
335 - | it until minOf(it + partSize, length())
|
307 + | internal fun ByteArray.chunk(partSize: Int): Sequence<ByteArray> = (0 until size step partSize).asSequence().map { start ->
|
308 + | copyOfRange(start, minOf(start + partSize, size))
|
336 309 | }
|
337 310 |
|
338 311 | internal suspend fun s3WithAllEngines(block: suspend (S3Client) -> Unit) {
|
339 312 | withAllEngines { engine ->
|
340 313 | S3Client {
|
341 314 | region = S3TestUtils.DEFAULT_REGION
|
342 315 | httpClient = engine
|
343 316 | }.use {
|
344 317 | try {
|
345 318 | block(it)
|