AWS SDK

AWS SDK

rev. beaf1e7be46b779a1dedd92bc48143dba23aff97..a483b08a5c5215e3404cafb73a660d0a9cd360fc (ignoring whitespace)

Files changed:

tmp-codegen-diff/services/s3/e2eTest/src/commonMain/kotlin/aws/sdk/kotlin/services/s3/S3PresignerTest.kt

Renamed from tmp-codegen-diff/services/s3/e2eTest/src/S3PresignerTest.kt

@@ -1,1 +104,108 @@
    1      1   
/*
    2      2   
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
    3      3   
 * SPDX-License-Identifier: Apache-2.0
    4      4   
 */
    5      5   
package aws.sdk.kotlin.e2etest
    6      6   
           7  +
import aws.sdk.kotlin.e2etest.S3TestUtils.responseCodeFromPut
    7      8   
import aws.sdk.kotlin.services.s3.S3Client
    8      9   
import aws.sdk.kotlin.services.s3.model.DeleteObjectRequest
    9     10   
import aws.sdk.kotlin.services.s3.model.GetObjectRequest
   10     11   
import aws.sdk.kotlin.services.s3.model.PutObjectRequest
   11     12   
import aws.sdk.kotlin.services.s3.presigners.presignDeleteObject
   12     13   
import aws.sdk.kotlin.services.s3.presigners.presignGetObject
   13     14   
import aws.sdk.kotlin.services.s3.presigners.presignPutObject
   14     15   
import aws.sdk.kotlin.testing.PRINTABLE_CHARS
   15     16   
import aws.sdk.kotlin.testing.withAllEngines
   16     17   
import aws.smithy.kotlin.runtime.content.decodeToString
   17     18   
import aws.smithy.kotlin.runtime.http.SdkHttpClient
   18     19   
import aws.smithy.kotlin.runtime.http.complete
   19     20   
import aws.smithy.kotlin.runtime.http.toByteStream
   20         -
import kotlinx.coroutines.*
   21         -
import kotlinx.coroutines.test.runTest
   22         -
import org.junit.jupiter.api.AfterAll
   23         -
import org.junit.jupiter.api.BeforeAll
   24         -
import org.junit.jupiter.api.TestInstance
          21  +
import aws.smithy.kotlin.runtime.io.use
          22  +
import aws.smithy.kotlin.runtime.testing.AfterAll
          23  +
import aws.smithy.kotlin.runtime.testing.BeforeAll
          24  +
import kotlinx.coroutines.runBlocking
          25  +
import kotlin.jvm.JvmStatic
   25     26   
import kotlin.test.Test
   26     27   
import kotlin.test.assertEquals
   27     28   
import kotlin.time.Duration.Companion.seconds
   28     29   
   29         -
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
   30     30   
class S3PresignerTest {
   31         -
    private val client = S3Client {
   32         -
        region = S3TestUtils.DEFAULT_REGION
   33         -
    }
   34         -
          31  +
    companion object {
          32  +
        private lateinit var client: S3Client
   35     33   
        private lateinit var testBucket: String
   36     34   
   37     35   
        @BeforeAll
   38         -
    fun createResources(): Unit = runBlocking {
   39         -
        testBucket = S3TestUtils.getTestBucket(client)
          36  +
        @JvmStatic
          37  +
        fun setup() = runBlocking {
          38  +
            client = S3Client {
          39  +
                region = S3TestUtils.DEFAULT_REGION
          40  +
            }
          41  +
            testBucket = S3TestUtils.getOrCreateSharedBucket(client)
   40     42   
        }
   41     43   
   42     44   
        @AfterAll
          45  +
        @JvmStatic
   43     46   
        fun cleanup(): Unit = runBlocking {
   44         -
        S3TestUtils.deleteBucketAndAllContents(client, testBucket)
          47  +
            S3TestUtils.cleanupSharedBucket(client)
   45     48   
            client.close()
   46     49   
        }
          50  +
    }
   47     51   
   48     52   
    private suspend fun testPresign(client: S3Client) {
   49     53   
        val contents = "presign-test"
   50     54   
        val keyName = "foo$PRINTABLE_CHARS"
   51     55   
   52     56   
        withAllEngines { engine ->
   53     57   
            val httpClient = SdkHttpClient(engine)
   54     58   
   55     59   
            // PUT
   56     60   
            val unsignedPutRequest = PutObjectRequest {
   57     61   
                bucket = testBucket
   58     62   
                key = keyName
   59     63   
            }
   60     64   
            val presignedPutRequest = client.presignPutObject(unsignedPutRequest, 60.seconds)
   61     65   
   62         -
            S3TestUtils.responseCodeFromPut(presignedPutRequest, contents)
          66  +
            responseCodeFromPut(engine, presignedPutRequest, contents)
   63     67   
   64     68   
            // GET
   65     69   
            val unsignedGetRequest = GetObjectRequest {
   66     70   
                bucket = testBucket
   67     71   
                key = keyName
   68     72   
            }
   69     73   
            val presignedGetRequest = client.presignGetObject(unsignedGetRequest, 60.seconds)
   70     74   
   71     75   
            val call = httpClient.call(presignedGetRequest)
   72     76   
            val body = call.response.body.toByteStream()?.decodeToString()
   73     77   
            call.complete()
   74     78   
            assertEquals(200, call.response.status.value)
   75     79   
            assertEquals(contents, body)
   76     80   
   77     81   
            // DELETE
   78     82   
            val unsignedDeleteRequest = DeleteObjectRequest {
   79     83   
                bucket = testBucket
   80     84   
                key = keyName
   81     85   
            }
   82     86   
            val presignedDeleteObject = client.presignDeleteObject(unsignedDeleteRequest, 60.seconds)
   83     87   
   84     88   
            val deleteCall = httpClient.call(presignedDeleteObject)
   85     89   
            deleteCall.complete()
   86     90   
            assertEquals(204, deleteCall.response.status.value)
   87     91   
        }
   88     92   
    }
   89     93   
   90     94   
    @Test
   91         -
    fun testPresignNormal() = runTest {
          95  +
    fun testPresignNormal() = runBlocking {
   92     96   
        S3Client {
   93     97   
            region = S3TestUtils.DEFAULT_REGION
   94     98   
        }.use { testPresign(it) }
   95     99   
    }
   96    100   
   97    101   
    @Test
   98    102   
    fun testPresignWithForcePathStyle() = runBlocking {
   99    103   
        S3Client {
  100    104   
            region = S3TestUtils.DEFAULT_REGION
  101    105   
            forcePathStyle = true

tmp-codegen-diff/services/s3/e2eTest/src/commonMain/kotlin/aws/sdk/kotlin/services/s3/S3TestUtils.kt

Renamed from tmp-codegen-diff/services/s3/e2eTest/src/S3TestUtils.kt

@@ -1,1 +221,256 @@
    6      6   
    7      7   
import aws.sdk.kotlin.services.s3.*
    8      8   
import aws.sdk.kotlin.services.s3.S3Client
    9      9   
import aws.sdk.kotlin.services.s3.model.*
   10     10   
import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint
   11     11   
import aws.sdk.kotlin.services.s3.model.ExpirationStatus
   12     12   
import aws.sdk.kotlin.services.s3.model.LifecycleRule
   13     13   
import aws.sdk.kotlin.services.s3.paginators.listObjectsV2Paginated
   14     14   
import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketExists
   15     15   
import aws.sdk.kotlin.services.s3.waiters.waitUntilBucketNotExists
   16         -
import aws.sdk.kotlin.services.s3control.*
   17         -
import aws.sdk.kotlin.services.s3control.model.*
   18     16   
import aws.sdk.kotlin.services.sts.StsClient
          17  +
import aws.smithy.kotlin.runtime.http.HttpBody
          18  +
import aws.smithy.kotlin.runtime.http.HttpMethod
          19  +
import aws.smithy.kotlin.runtime.http.SdkHttpClient
          20  +
import aws.smithy.kotlin.runtime.http.complete
          21  +
import aws.smithy.kotlin.runtime.http.engine.HttpClientEngine
   19     22   
import aws.smithy.kotlin.runtime.http.request.HttpRequest
          23  +
import aws.smithy.kotlin.runtime.io.use
   20     24   
import aws.smithy.kotlin.runtime.text.ensurePrefix
          25  +
import aws.smithy.kotlin.runtime.util.Uuid
   21     26   
import kotlinx.coroutines.*
   22     27   
import kotlinx.coroutines.flow.*
          28  +
import kotlinx.coroutines.sync.Mutex
          29  +
import kotlinx.coroutines.sync.withLock
   23     30   
import kotlinx.coroutines.withTimeout
   24         -
import java.io.OutputStreamWriter
   25         -
import java.net.URL
   26         -
import java.util.*
   27         -
import javax.net.ssl.HttpsURLConnection
   28     31   
import kotlin.time.Duration.Companion.seconds
   29     32   
   30     33   
object S3TestUtils {
   31     34   
   32     35   
    const val DEFAULT_REGION = "us-west-2"
   33     36   
   34     37   
    // The E2E test account only has permission to operate on buckets with the prefix "s3-test-bucket-"
   35     38   
    private const val TEST_BUCKET_PREFIX = "s3-test-bucket-"
   36     39   
          40  +
    private var sharedBucket: String? = null
          41  +
    private val bucketMutex = Mutex()
          42  +
          43  +
    private val sharedDirectoryBuckets: MutableMap<String, String> = mutableMapOf()
          44  +
    private val directoryBucketMutex = Mutex()
          45  +
          46  +
    suspend fun getOrCreateSharedBucket(client: S3Client, region: String = DEFAULT_REGION): String = sharedBucket ?: bucketMutex.withLock {
          47  +
        sharedBucket ?: getTestBucket(client, region).also { sharedBucket = it }
          48  +
    }
          49  +
          50  +
    suspend fun cleanupSharedBucket(client: S3Client) {
          51  +
        sharedBucket?.let { bucket ->
          52  +
            deleteBucketContents(client, bucket)
          53  +
        }
          54  +
    }
          55  +
          56  +
    suspend fun getOrCreateSharedDirectoryBuckets(client: S3Client, suffix: String): List<String> = directoryBucketMutex.withLock {
          57  +
        (0 until 3).map { index ->
          58  +
            val key = "$suffix:$index"
          59  +
            sharedDirectoryBuckets[key] ?: getTestDirectoryBucket(client, suffix).also {
          60  +
                sharedDirectoryBuckets[key] = it
          61  +
            }
          62  +
        }
          63  +
    }
          64  +
          65  +
    suspend fun cleanupSharedDirectoryBuckets(client: S3Client, suffix: String) {
          66  +
        (0 until 3).forEach { index ->
          67  +
            val key = "$suffix:$index"
          68  +
            sharedDirectoryBuckets[key]?.let { bucket ->
          69  +
                deleteBucketContents(client, bucket)
          70  +
            }
          71  +
        }
          72  +
    }
          73  +
   37     74   
    private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
   38     75   
    private const val S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX = "--x-s3"
   39     76   
   40     77   
    suspend fun getTestBucket(
   41     78   
        client: S3Client,
   42     79   
        region: String? = null,
   43     80   
        accountId: String? = null,
   44     81   
    ): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX, region, accountId)
   45     82   
   46     83   
    suspend fun getBucketWithPrefix(
   47     84   
        client: S3Client,
   48     85   
        prefix: String,
   49     86   
        region: String? = null,
   50     87   
        accountId: String? = null,
   51     88   
    ): String = withTimeout(60.seconds) {
   52     89   
        val buckets = client.listBuckets()
   53     90   
            .buckets
   54     91   
            ?.mapNotNull { it.name }
   55     92   
   56     93   
        var testBucket = buckets?.firstOrNull { bucketName ->
   57     94   
            bucketName.startsWith(prefix) &&
   58     95   
                region?.let {
   59     96   
                    client.getBucketLocation {
   60     97   
                        bucket = bucketName
   61     98   
                        expectedBucketOwner = accountId
   62     99   
                    }.locationConstraint?.value == region
   63    100   
                } ?: true
   64    101   
        }
   65    102   
   66    103   
        if (testBucket == null) {
   67         -
            testBucket = prefix + UUID.randomUUID()
         104  +
            testBucket = prefix + Uuid.random().toString()
   68    105   
            println("Creating S3 bucket: $testBucket")
   69    106   
   70    107   
            client.createBucket {
   71    108   
                bucket = testBucket
   72    109   
                createBucketConfiguration {
   73    110   
                    locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!)
   74    111   
                }
   75    112   
            }
   76    113   
   77    114   
            client.waitUntilBucketExists { bucket = testBucket }
   78    115   
        } else {
   79    116   
            println("Using existing S3 bucket: $testBucket")
   80    117   
        }
   81    118   
   82    119   
        client.putBucketLifecycleConfiguration {
   83    120   
            bucket = testBucket
   84    121   
            lifecycleConfiguration {
   85    122   
                rules = listOf(
   86    123   
                    LifecycleRule {
   87    124   
                        expiration { days = 1 }
   88    125   
                        filter { this.prefix = "" }
   89    126   
                        status = ExpirationStatus.Enabled
   90    127   
                        id = "delete-old"
   91    128   
                    },
   92    129   
                )
   93    130   
            }
   94    131   
        }
   95    132   
   96    133   
        testBucket
   97    134   
    }
   98    135   
   99    136   
    suspend fun getTestDirectoryBucket(client: S3Client, suffix: String) = withTimeout(60.seconds) {
  100    137   
        var testBucket = client.listBuckets()
  101    138   
            .buckets
  102    139   
            ?.mapNotNull { it.name }
  103    140   
            ?.firstOrNull { it.startsWith(TEST_BUCKET_PREFIX) && it.endsWith(S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX) }
  104    141   
  105    142   
        if (testBucket == null) {
  106    143   
            // Adding S3 Express suffix surpasses the bucket name length limit... trim the UUID if needed
  107    144   
            testBucket = TEST_BUCKET_PREFIX +
  108         -
                UUID.randomUUID().toString().subSequence(0 until (S3_MAX_BUCKET_NAME_LENGTH - TEST_BUCKET_PREFIX.length - suffix.ensurePrefix("--").length)) +
         145  +
                Uuid.random().toString().subSequence(0 until (S3_MAX_BUCKET_NAME_LENGTH - TEST_BUCKET_PREFIX.length - suffix.ensurePrefix("--").length)) +
  109    146   
                suffix.ensurePrefix("--")
  110    147   
  111    148   
            println("Creating S3 Express directory bucket: $testBucket")
  112    149   
  113    150   
            val availabilityZone = testBucket // s3-test-bucket-UUID--use1-az4--x-s3
  114    151   
                .removeSuffix(S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX) // s3-test-bucket-UUID--use1-az4
  115    152   
                .substringAfterLast("--") // use1-az4
  116    153   
  117    154   
            client.createBucket {
  118    155   
                bucket = testBucket
  119    156   
                createBucketConfiguration {
  120    157   
                    location = LocationInfo {
  121    158   
                        type = LocationType.AvailabilityZone
  122    159   
                        name = availabilityZone
  123    160   
                    }
  124    161   
                    bucket = BucketInfo {
  125    162   
                        type = BucketType.Directory
  126    163   
                        dataRedundancy = DataRedundancy.SingleAvailabilityZone
  127    164   
                    }
  128    165   
                }
  129    166   
            }
         167  +
        } else {
         168  +
            println("Using existing S3 Express directory bucket: $testBucket")
  130    169   
        }
         170  +
         171  +
        client.putBucketLifecycleConfiguration {
         172  +
            bucket = testBucket
         173  +
            lifecycleConfiguration {
         174  +
                rules = listOf(
         175  +
                    LifecycleRule {
         176  +
                        expiration { days = 1 }
         177  +
                        filter { this.prefix = "" }
         178  +
                        status = ExpirationStatus.Enabled
         179  +
                        id = "delete-old"
         180  +
                    },
         181  +
                )
         182  +
            }
         183  +
        }
         184  +
  131    185   
        testBucket
  132    186   
    }
  133    187   
  134    188   
    suspend fun deleteBucketAndAllContents(client: S3Client, bucketName: String): Unit = coroutineScope {
  135    189   
        deleteBucketContents(client, bucketName)
  136    190   
  137    191   
        try {
         192  +
            println("Deleting S3 bucket: $bucketName")
  138    193   
            client.deleteBucket { bucket = bucketName }
  139    194   
  140    195   
            client.waitUntilBucketNotExists {
  141    196   
                bucket = bucketName
  142    197   
            }
  143    198   
        } catch (ex: Exception) {
  144    199   
            println("Failed to delete bucket: $bucketName")
  145    200   
            throw ex
  146    201   
        }
  147    202   
    }
  148    203   
  149         -
    suspend fun deleteBucketContents(client: S3Client, bucketName: String): Unit = coroutineScope {
         204  +
    suspend fun deleteBucketContents(client: S3Client, bucketName: String) = coroutineScope {
  150    205   
        val scope = this
  151    206   
  152    207   
        try {
  153         -
            println("Deleting S3 buckets contents: $bucketName")
         208  +
            println("Deleting contents of S3 bucket: $bucketName")
  154    209   
            val dispatcher = Dispatchers.Default.limitedParallelism(64)
  155    210   
            val jobs = mutableListOf<Job>()
  156    211   
  157    212   
            client.listObjectsV2Paginated { bucket = bucketName }
  158    213   
                .mapNotNull { it.contents }
  159    214   
                .collect { contents ->
  160    215   
                    val job = scope.launch(dispatcher) {
  161    216   
                        client.deleteObjects {
  162    217   
                            bucket = bucketName
  163    218   
                            delete {
  164    219   
                                objects = contents.mapNotNull(Object::key).map { ObjectIdentifier { key = it } }
  165    220   
                            }
  166    221   
                        }
  167    222   
                    }
  168    223   
                    jobs.add(job)
  169    224   
                }
  170    225   
  171    226   
            jobs.joinAll()
  172    227   
        } catch (ex: Exception) {
  173    228   
            println("Failed to delete buckets contents: $bucketName")
  174    229   
            throw ex
  175    230   
        }
  176    231   
    }
  177    232   
  178         -
    fun responseCodeFromPut(presignedRequest: HttpRequest, content: String): Int {
  179         -
        val url = URL(presignedRequest.url.toString())
  180         -
        val connection: HttpsURLConnection = url.openConnection() as HttpsURLConnection
  181         -
        presignedRequest.headers.forEach { key, values ->
  182         -
            connection.setRequestProperty(key, values.first())
  183         -
        }
  184         -
  185         -
        connection.doOutput = true
  186         -
        connection.requestMethod = "PUT"
  187         -
        val out = OutputStreamWriter(connection.outputStream)
  188         -
        out.write(content)
  189         -
        out.close()
  190         -
  191         -
        if (connection.errorStream != null) {
  192         -
            error("request failed: ${connection.errorStream?.bufferedReader()?.readText()}")
  193         -
        }
         233  +
    suspend fun responseCodeFromPut(engine: HttpClientEngine, presignedRequest: HttpRequest, content: String): Int {
         234  +
        val request = HttpRequest(
         235  +
            method = HttpMethod.PUT,
         236  +
            url = presignedRequest.url,
         237  +
            headers = presignedRequest.headers,
         238  +
            body = HttpBody.fromBytes(content.encodeToByteArray()),
         239  +
        )
  194    240   
  195         -
        return connection.responseCode
         241  +
        val call = SdkHttpClient(engine).call(request)
         242  +
        val statusCode = call.response.status.value
         243  +
        call.complete()
         244  +
        return statusCode
  196    245   
    }
  197    246   
  198    247   
    internal suspend fun getAccountId(): String {
  199         -
        println("Getting account ID")
  200         -
  201    248   
        val accountId = StsClient {
  202    249   
            region = "us-west-2"
  203    250   
        }.use {
  204    251   
            it.getCallerIdentity().account
  205    252   
        }
  206    253   
  207    254   
        return checkNotNull(accountId) { "Unable to get AWS account ID" }
  208    255   
    }
  209         -
  210         -
    internal suspend fun deleteMultiPartUploads(client: S3Client, bucketName: String) {
  211         -
        client.listMultipartUploads {
  212         -
            bucket = bucketName
  213         -
        }.uploads?.forEach { upload ->
  214         -
            client.abortMultipartUpload {
  215         -
                bucket = bucketName
  216         -
                key = upload.key
  217         -
                uploadId = upload.uploadId
  218         -
            }
  219         -
        }
  220         -
    }
  221    256   
}

tmp-codegen-diff/services/s3/e2eTest/src/jvmMain/kotlin/aws/sdk/kotlin/services/s3/ConnectionResetTest.kt

Renamed from tmp-codegen-diff/services/s3/e2eTest/src/ConnectionResetTest.kt

@@ -1,1 +79,74 @@
    1      1   
/*
    2      2   
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
    3      3   
 * SPDX-License-Identifier: Apache-2.0
    4      4   
 */
    5      5   
package aws.sdk.kotlin.e2etest
    6      6   
    7      7   
import aws.sdk.kotlin.services.s3.S3Client
    8      8   
import aws.sdk.kotlin.services.s3.model.PutObjectRequest
    9      9   
import aws.smithy.kotlin.runtime.content.ByteStream
   10     10   
import aws.smithy.kotlin.runtime.http.HttpException
   11         -
import aws.smithy.kotlin.runtime.util.Uuid
   12     11   
import kotlinx.coroutines.async
   13     12   
import kotlinx.coroutines.awaitAll
   14     13   
import kotlinx.coroutines.delay
   15         -
import kotlinx.coroutines.invoke
   16     14   
import kotlinx.coroutines.runBlocking
   17         -
import org.junit.jupiter.api.AfterAll
   18         -
import org.junit.jupiter.api.BeforeAll
   19         -
import org.junit.jupiter.api.TestInstance
   20         -
import java.io.IOException
          15  +
import kotlin.test.AfterTest
          16  +
import kotlin.test.BeforeTest
   21     17   
import kotlin.test.Test
   22     18   
import kotlin.time.Duration.Companion.seconds
   23     19   
   24     20   
/**
   25     21   
 * Reproduces "unexpected end of stream" errors as seen in https://github.com/aws/aws-sdk-kotlin/issues/1214
   26     22   
 * and ensures they are resolved by OkHttp's retryOnConnectionFailure option
   27     23   
 */
   28         -
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
   29     24   
class ConnectionResetTest {
   30     25   
    private val client = S3Client {
   31     26   
        region = S3TestUtils.DEFAULT_REGION
   32     27   
    }
   33     28   
   34     29   
    private lateinit var testBucket: String
   35     30   
   36         -
    @BeforeAll
   37         -
    fun createResources(): Unit = runBlocking {
   38         -
        testBucket = S3TestUtils.getTestBucket(client)
          31  +
    @BeforeTest
          32  +
    fun createResources() = runBlocking {
          33  +
        testBucket = S3TestUtils.getOrCreateSharedBucket(client)
   39     34   
    }
   40     35   
   41         -
    @AfterAll
          36  +
    @AfterTest
   42     37   
    fun cleanup() = runBlocking {
   43         -
        S3TestUtils.deleteBucketAndAllContents(client, testBucket)
          38  +
        S3TestUtils.cleanupSharedBucket(client)
   44     39   
    }
   45     40   
   46     41   
    @Test
   47         -
    fun testConnectionResetDoesntThrow(): Unit = runBlocking {
          42  +
    fun testConnectionResetDoesntThrow() = runBlocking {
   48     43   
        // Launch multiple coroutines to populate connection pool
   49     44   
        val jobs = (1..10).map {
   50     45   
            async { client.putTestObject() }
   51     46   
        }
   52     47   
        jobs.awaitAll()
   53     48   
        // Connections are now idle in the pool
   54     49   
   55     50   
        // Wait for S3 to close stale connections
   56     51   
        delay(7.seconds)
   57     52   
   58     53   
        // Try to re-use a connection
   59     54   
        client.putTestObject()
   60     55   
    }
   61     56   
   62     57   
    suspend fun S3Client.putTestObject() {
   63     58   
        val putObjectRequest = PutObjectRequest {
   64     59   
            bucket = testBucket
   65         -
            key = Uuid.random().toString()
          60  +
            key = (0..Int.MAX_VALUE).random().toString()
   66     61   
            body = ByteStream.fromString("Content")
   67     62   
        }
   68     63   
   69     64   
        try {
   70         -
            client.putObject(putObjectRequest)
          65  +
            putObject(putObjectRequest)
   71     66   
        } catch (e: HttpException) {
   72         -
            if (e.cause is IOException && e.cause?.message?.contains("unexpected end of stream") == true) {
   73         -
                throw RetryOnConnectionFailureException("SDK unexpectedly threw java.io.IOException which should have been retried by OkHttp's retryOnConnectionFailure feature", e)
          67  +
            if (e.cause?.message?.contains("unexpected end of stream") == true) {
          68  +
                throw RetryOnConnectionFailureException("SDK unexpectedly threw exception which should have been retried by HTTP engine's retry feature", e)
   74     69   
            }
   75     70   
        }
   76     71   
    }
   77     72   
}
   78     73   
   79     74   
class RetryOnConnectionFailureException(message: String, cause: Exception? = null) : Exception(message, cause)

tmp-codegen-diff/services/s3/e2eTest/src/jvmMain/kotlin/aws/sdk/kotlin/services/s3/S3IntegrationTestJVM.kt

@@ -0,1 +0,63 @@
           1  +
/*
           2  +
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
           3  +
 * SPDX-License-Identifier: Apache-2.0
           4  +
 */
           5  +
package aws.sdk.kotlin.e2etest
           6  +
           7  +
import aws.sdk.kotlin.e2etest.S3TestUtils.getOrCreateSharedBucket
           8  +
import aws.sdk.kotlin.services.s3.S3Client
           9  +
import aws.sdk.kotlin.services.s3.model.*
          10  +
import aws.sdk.kotlin.services.s3.putObject
          11  +
import aws.smithy.kotlin.runtime.content.ByteStream
          12  +
import aws.smithy.kotlin.runtime.content.decodeToString
          13  +
import aws.smithy.kotlin.runtime.content.fromFile
          14  +
import aws.smithy.kotlin.runtime.testing.RandomTempFile
          15  +
import kotlinx.coroutines.runBlocking
          16  +
import kotlinx.coroutines.withTimeout
          17  +
import kotlin.test.AfterTest
          18  +
import kotlin.test.BeforeTest
          19  +
import kotlin.test.Test
          20  +
import kotlin.test.assertEquals
          21  +
import kotlin.time.Duration.Companion.seconds
          22  +
          23  +
class S3IntegrationTestJVM {
          24  +
    private lateinit var client: S3Client
          25  +
    private lateinit var testBucket: String
          26  +
          27  +
    @BeforeTest
          28  +
    fun setUp() = runBlocking {
          29  +
        client = S3Client { region = "us-west-2" }
          30  +
        testBucket = getOrCreateSharedBucket(client)
          31  +
    }
          32  +
          33  +
    @AfterTest
          34  +
    fun cleanUp() = runBlocking {
          35  +
        client.close()
          36  +
    }
          37  +
          38  +
    @Test
          39  +
    fun testPutObjectFromFile() = runBlocking<Unit> {
          40  +
        val tempFile = RandomTempFile(1024)
          41  +
        val keyName = "put-obj-from-file.txt"
          42  +
          43  +
        // This test fails sporadically (by never completing)
          44  +
        // see https://github.com/awslabs/aws-sdk-kotlin/issues/282
          45  +
        withTimeout(5.seconds) {
          46  +
            client.putObject {
          47  +
                bucket = testBucket
          48  +
                key = keyName
          49  +
                body = ByteStream.fromFile(tempFile)
          50  +
            }
          51  +
        }
          52  +
          53  +
        val req = GetObjectRequest {
          54  +
            bucket = testBucket
          55  +
            key = keyName
          56  +
        }
          57  +
        val roundTrippedContents = client.getObject(req) { it.body?.decodeToString() }
          58  +
          59  +
        val contents = tempFile.readText()
          60  +
        assertEquals(contents, roundTrippedContents)
          61  +
        tempFile.delete()
          62  +
    }
          63  +
}

tmp-codegen-diff/services/sesv2/e2eTest/src/commonMain/kotlin/Sigv4aTest.kt

Renamed from tmp-codegen-diff/services/sesv2/e2eTest/src/Sigv4aTest.kt

@@ -1,1 +53,54 @@
    4      4   
 */
    5      5   
    6      6   
import aws.sdk.kotlin.services.sesv2.SesV2Client
    7      7   
import aws.sdk.kotlin.services.sesv2.sendEmail
    8      8   
import aws.smithy.kotlin.runtime.auth.awssigning.crt.CrtAwsSigner
    9      9   
import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext
   10     10   
import aws.smithy.kotlin.runtime.http.HttpException
   11     11   
import aws.smithy.kotlin.runtime.http.auth.SigV4AsymmetricAuthScheme
   12     12   
import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor
   13     13   
import aws.smithy.kotlin.runtime.http.request.HttpRequest
   14         -
import kotlinx.coroutines.runBlocking
          14  +
import aws.smithy.kotlin.runtime.io.use
          15  +
import kotlinx.coroutines.test.runTest
   15     16   
import kotlin.test.Test
   16     17   
import kotlin.test.assertContains
   17     18   
import kotlin.test.assertEquals
   18     19   
import kotlin.test.assertFailsWith
   19     20   
import kotlin.test.assertNotNull
   20     21   
   21     22   
class Sigv4aTest {
   22     23   
    @Test
   23         -
    fun testSigv4a() = runBlocking {
          24  +
    fun testSigv4a() = runTest {
   24     25   
        val interceptor = RequestCapturingInterceptor()
   25     26   
   26     27   
        SesV2Client.fromEnvironment {
   27     28   
            retryStrategy {
   28     29   
                maxAttempts = 1 // The call is intended to fail, no sense trying more than once
   29     30   
            }
   30     31   
   31     32   
            interceptors += interceptor
   32     33   
   33     34   
            authSchemes = listOf(SigV4AsymmetricAuthScheme(CrtAwsSigner, "ses"))

tmp-codegen-diff/services/sts/e2eTest/src/commonMain/kotlin/aws/sdk/kotlin/services/sts/STSPresignerTest.kt

Renamed from tmp-codegen-diff/services/sts/e2eTest/src/STSPresignerTest.kt

@@ -1,1 +39,38 @@
    2      2   
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
    3      3   
 * SPDX-License-Identifier: Apache-2.0
    4      4   
 */
    5      5   
package aws.sdk.kotlin.services.sts
    6      6   
    7      7   
import aws.sdk.kotlin.services.sts.model.GetCallerIdentityRequest
    8      8   
import aws.sdk.kotlin.services.sts.presigners.presignGetCallerIdentity
    9      9   
import aws.sdk.kotlin.testing.withAllEngines
   10     10   
import aws.smithy.kotlin.runtime.http.SdkHttpClient
   11     11   
import aws.smithy.kotlin.runtime.http.complete
   12         -
import kotlinx.coroutines.runBlocking
   13         -
import org.junit.jupiter.api.TestInstance
          12  +
import aws.smithy.kotlin.runtime.io.use
          13  +
import kotlinx.coroutines.test.runTest
   14     14   
import kotlin.test.Test
   15     15   
import kotlin.test.assertEquals
   16     16   
import kotlin.time.Duration.Companion.seconds
   17     17   
   18     18   
/**
   19     19   
 * Tests for presigner
   20     20   
 */
   21         -
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
   22     21   
class StsPresignerTest {
   23     22   
    @Test
   24         -
    fun testGetCallerIdentityPresigner() = runBlocking {
          23  +
    fun testGetCallerIdentityPresigner() = runTest {
   25     24   
        val req = GetCallerIdentityRequest { }
   26     25   
   27     26   
        val presignedRequest = StsClient { region = "us-west-2" }.use { sts ->
   28     27   
            sts.presignGetCallerIdentity(req, 60.seconds)
   29     28   
        }
   30     29   
   31     30   
        withAllEngines { engine ->
   32     31   
            val httpClient = SdkHttpClient(engine)
   33     32   
            val call = httpClient.call(presignedRequest)
   34     33   
            call.complete()

tmp-codegen-diff/services/transcribestreaming/e2eTest/src/commonMain/kotlin/aws/sdk/kotlin/services/transcribestreaming/TranscribeStreamingIntegrationTest.kt

Renamed from tmp-codegen-diff/services/transcribestreaming/e2eTest/src/TranscribeStreamingIntegrationTest.kt

@@ -1,1 +0,57 @@
    1      1   
/*
    2      2   
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
    3      3   
 * SPDX-License-Identifier: Apache-2.0
    4      4   
 */
    5      5   
package aws.sdk.kotlin.e2etest
    6      6   
    7      7   
import aws.sdk.kotlin.services.transcribestreaming.TranscribeStreamingClient
    8      8   
import aws.sdk.kotlin.services.transcribestreaming.model.*
    9         -
import kotlinx.coroutines.Dispatchers
           9  +
import aws.smithy.kotlin.runtime.io.use
          10  +
import aws.smithy.kotlin.runtime.testing.IgnoreNative
   10     11   
import kotlinx.coroutines.flow.Flow
   11         -
import kotlinx.coroutines.flow.flow
   12         -
import kotlinx.coroutines.flow.flowOn
   13         -
import kotlinx.coroutines.runBlocking
   14         -
import org.junit.jupiter.api.Test
   15         -
import org.junit.jupiter.api.TestInstance
   16         -
import java.io.File
   17         -
import java.nio.file.Paths
   18         -
import javax.sound.sampled.AudioSystem
          12  +
import kotlinx.coroutines.test.runTest
          13  +
import kotlin.test.Test
   19     14   
import kotlin.test.assertTrue
   20     15   
   21         -
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
   22     16   
class TranscribeStreamingIntegrationTest {
   23         -
          17  +
    @IgnoreNative // FIXME Implement native bidirectional streaming in CRT
   24     18   
    @Test
   25         -
    fun testTranscribeEventStream(): Unit = runBlocking {
   26         -
        val url = this::class.java.classLoader.getResource("hello-kotlin-8000.wav") ?: error("failed to load test resource")
   27         -
        val audioFile = Paths.get(url.toURI()).toFile()
   28         -
          19  +
    fun testTranscribeEventStream() = runTest {
   29     20   
        TranscribeStreamingClient { region = "us-west-2" }.use { client ->
   30         -
            val transcript = getTranscript(client, audioFile)
          21  +
            val transcript = getTranscript(client)
   31     22   
            assertTrue(transcript.startsWith("Hello from", true), "full transcript: $transcript")
   32     23   
        }
   33     24   
    }
   34     25   
}
   35     26   
   36         -
private const val FRAMES_PER_CHUNK = 4096
   37         -
   38         -
private fun audioStreamFromFile(file: File): Flow<AudioStream> {
   39         -
    val format = AudioSystem.getAudioFileFormat(file)
   40         -
    val ais = AudioSystem.getAudioInputStream(file)
   41         -
    val bytesPerFrame = ais.format.frameSize
   42         -
    println("audio stream format of $file: $format; bytesPerFrame=$bytesPerFrame")
   43         -
   44         -
    return flow {
   45         -
        while (true) {
   46         -
            val frameBuffer = ByteArray(FRAMES_PER_CHUNK * bytesPerFrame)
   47         -
            val rc = ais.read(frameBuffer)
   48         -
            if (rc <= 0) {
   49         -
                break
   50         -
            }
   51         -
   52         -
            val chunk = if (rc < frameBuffer.size) frameBuffer.sliceArray(0 until rc) else frameBuffer
   53         -
            val event = AudioStream.AudioEvent(
   54         -
                AudioEvent {
   55         -
                    audioChunk = chunk
   56         -
                },
   57         -
            )
   58         -
   59         -
            println("emitting event")
   60         -
            emit(event)
   61         -
        }
   62         -
    }.flowOn(Dispatchers.IO)
   63         -
}
   64         -
   65         -
private suspend fun getTranscript(client: TranscribeStreamingClient, audioFile: File): String {
          27  +
private suspend fun getTranscript(client: TranscribeStreamingClient): String {
   66     28   
    val req = StartStreamTranscriptionRequest {
   67     29   
        languageCode = LanguageCode.EnUs
   68     30   
        mediaSampleRateHertz = 8000
   69     31   
        mediaEncoding = MediaEncoding.Pcm
   70         -
        audioStream = audioStreamFromFile(audioFile)
          32  +
        audioStream = loadAudioStream()
   71     33   
    }
   72     34   
   73     35   
    val transcript = client.startStreamTranscription(req) { resp ->
   74     36   
        val fullMessage = StringBuilder()
   75     37   
        resp.transcriptResultStream?.collect { event ->
   76     38   
            when (event) {
   77     39   
                is TranscriptResultStream.TranscriptEvent -> {
   78     40   
                    event.value.transcript?.results?.forEach { result ->
   79     41   
                        val transcript = result.alternatives?.firstOrNull()?.transcript
   80     42   
                        println("received TranscriptEvent: isPartial=${result.isPartial}; transcript=$transcript")
   81     43   
                        if (!result.isPartial) {
   82     44   
                            transcript?.let { fullMessage.append(it) }
   83     45   
                        }
   84     46   
                    }
   85     47   
                }
   86     48   
                else -> error("unknown event $event")
   87     49   
            }
   88     50   
        }
   89     51   
        fullMessage.toString()
   90     52   
    }
   91     53   
   92     54   
    return transcript
   93     55   
}
          56  +
          57  +
expect fun loadAudioStream(): Flow<AudioStream>

tmp-codegen-diff/services/transcribestreaming/e2eTest/src/jvmMain/kotlin/aws/sdk/kotlin/services/transcribestreaming/TranscribeStreamingIntegrationTestJvm.kt

@@ -0,1 +0,46 @@
           1  +
/*
           2  +
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
           3  +
 * SPDX-License-Identifier: Apache-2.0
           4  +
 */
           5  +
package aws.sdk.kotlin.e2etest
           6  +
           7  +
import aws.sdk.kotlin.services.transcribestreaming.model.AudioEvent
           8  +
import aws.sdk.kotlin.services.transcribestreaming.model.AudioStream
           9  +
import kotlinx.coroutines.Dispatchers
          10  +
import kotlinx.coroutines.flow.Flow
          11  +
import kotlinx.coroutines.flow.flow
          12  +
import kotlinx.coroutines.flow.flowOn
          13  +
import java.nio.file.Paths
          14  +
import javax.sound.sampled.AudioSystem
          15  +
          16  +
private const val FRAMES_PER_CHUNK = 4096
          17  +
          18  +
actual fun loadAudioStream(): Flow<AudioStream> {
          19  +
    val url = object {}.javaClass.classLoader.getResource("hello-kotlin-8000.wav") ?: error("failed to load test resource")
          20  +
    val audioFile = Paths.get(url.toURI()).toFile()
          21  +
          22  +
    val format = AudioSystem.getAudioFileFormat(audioFile)
          23  +
    val ais = AudioSystem.getAudioInputStream(audioFile)
          24  +
    val bytesPerFrame = ais.format.frameSize
          25  +
    println("audio stream format of $audioFile: $format; bytesPerFrame=$bytesPerFrame")
          26  +
          27  +
    return flow {
          28  +
        while (true) {
          29  +
            val frameBuffer = ByteArray(FRAMES_PER_CHUNK * bytesPerFrame)
          30  +
            val rc = ais.read(frameBuffer)
          31  +
            if (rc <= 0) {
          32  +
                break
          33  +
            }
          34  +
          35  +
            val chunk = if (rc < frameBuffer.size) frameBuffer.sliceArray(0 until rc) else frameBuffer
          36  +
            val event = AudioStream.AudioEvent(
          37  +
                AudioEvent {
          38  +
                    audioChunk = chunk
          39  +
                },
          40  +
            )
          41  +
          42  +
            println("emitting event")
          43  +
            emit(event)
          44  +
        }
          45  +
    }.flowOn(Dispatchers.IO)
          46  +
}

tmp-codegen-diff/services/transcribestreaming/e2eTest/src/nativeMain/kotlin/aws/sdk/kotlin/services/transcribestreaming/TranscribeStreamingIntegrationTestNative.kt

@@ -0,1 +0,73 @@
           1  +
/*
           2  +
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
           3  +
 * SPDX-License-Identifier: Apache-2.0
           4  +
 */
           5  +
package aws.sdk.kotlin.e2etest
           6  +
           7  +
import aws.sdk.kotlin.services.transcribestreaming.model.AudioEvent
           8  +
import aws.sdk.kotlin.services.transcribestreaming.model.AudioStream
           9  +
import aws.smithy.kotlin.runtime.util.PlatformProvider
          10  +
import kotlinx.coroutines.flow.Flow
          11  +
import kotlinx.coroutines.flow.flow
          12  +
          13  +
private const val FRAMES_PER_CHUNK = 4096
          14  +
          15  +
actual fun loadAudioStream(): Flow<AudioStream> = flow {
          16  +
    // Read the WAV file from test resources
          17  +
    val audioData = PlatformProvider.System.readFileOrNull("e2eTest/test-resources/hello-kotlin-8000.wav")
          18  +
        ?: error("failed to load test resource: hello-kotlin-8000.wav")
          19  +
          20  +
    // Parse WAV header to find data chunk
          21  +
    // WAV format: RIFF header (12 bytes) + format chunk + data chunk
          22  +
    var offset = 12 // Skip "RIFF" + size + "WAVE"
          23  +
    var dataOffset = -1
          24  +
    var dataSize = 0
          25  +
          26  +
    while (offset < audioData.size - 8) {
          27  +
        val chunkId = audioData.sliceArray(offset until offset + 4).decodeToString()
          28  +
        val chunkSize = audioData[offset + 4].toInt() and 0xFF or
          29  +
            ((audioData[offset + 5].toInt() and 0xFF) shl 8) or
          30  +
            ((audioData[offset + 6].toInt() and 0xFF) shl 16) or
          31  +
            ((audioData[offset + 7].toInt() and 0xFF) shl 24)
          32  +
          33  +
        if (chunkId == "data") {
          34  +
            dataOffset = offset + 8
          35  +
            dataSize = chunkSize
          36  +
            break
          37  +
        }
          38  +
        offset += 8 + chunkSize
          39  +
    }
          40  +
          41  +
    if (dataOffset == -1) {
          42  +
        error("Could not find data chunk in WAV file")
          43  +
    }
          44  +
          45  +
    val pcmData = audioData.sliceArray(dataOffset until dataOffset + dataSize)
          46  +
    println("audio stream loaded: ${pcmData.size} bytes of PCM data, dataOffset=$dataOffset, dataSize=$dataSize")
          47  +
          48  +
    // Use same chunk size as JVM (FRAMES_PER_CHUNK * bytesPerFrame)
          49  +
    // For 8kHz 16-bit mono: bytesPerFrame = 2
          50  +
    val bytesPerFrame = 2
          51  +
    val chunkSize = FRAMES_PER_CHUNK * bytesPerFrame
          52  +
    println("Will emit chunks of size $chunkSize bytes")
          53  +
          54  +
    var pos = 0
          55  +
    var eventCount = 0
          56  +
    while (pos < pcmData.size) {
          57  +
        val size = minOf(chunkSize, pcmData.size - pos)
          58  +
        val chunk = pcmData.sliceArray(pos until pos + size)
          59  +
          60  +
        val event = AudioStream.AudioEvent(
          61  +
            AudioEvent {
          62  +
                audioChunk = chunk
          63  +
            },
          64  +
        )
          65  +
          66  +
        eventCount++
          67  +
        println("emitting event #$eventCount (${chunk.size} bytes)")
          68  +
        emit(event)
          69  +
        pos += size
          70  +
    }
          71  +
          72  +
    println("Total events emitted: $eventCount")
          73  +
}