AWS CLI
Warning Before using check that you have properly set the AWS_PROFILE
env variable to some profile existing in ~/.aws/credentials
, or comfortable using the aws default profile
as
$ export AWS_PROFILE=analytics-storm
Get kinesis decoded data
aws kinesis get-records --shard-iterator some_shard_hash --profile some-profile | jq ".Records [] .Data" | tr -d ' "' | base64 -d
$ aws kinesis describe-stream --stream-name stream_name --profile user2
{
"StreamDescription": {
"RetentionPeriodHours": 24,
"StreamName": "stream_name",
"Shards": [
{
"ShardId": "shardId-000000000000",
"HashKeyRange": {
"EndingHashKey": "whateverhashkey",
"StartingHashKey": "0"
},
"SequenceNumberRange": {
"StartingSequenceNumber": "whateversequencenumber"
}
},
{
"ShardId": "shardId-000000000001",
"HashKeyRange": {
"EndingHashKey": "whateverhashke",
"StartingHashKey": "whatevernumber"
},
"SequenceNumberRange": {
"StartingSequenceNumber": "somenumber"
}
}
],
"StreamARN": "arn:aws:kinesis:region:accountNumber:stream/stream_name",
"EnhancedMonitoring": [
{
"ShardLevelMetrics": []
}
],
"StreamStatus": "ACTIVE"
}
}
$ aws kinesis get-shard-iterator --stream-name tvmetrix --shard-id shardId-000000000001 --shard-iterator-type LATEST
{
"ShardIterator": "whateversharditeratorwithnumberslettersandslashes "
}
get shard iterator for specific date (take into account the retention policy for your kinesis service)
$ aws kinesis get-shard-iterator --stream-name tvmetrix --shard-id shardId-000000000001 --shard-iterator-type AT_TIMESTAMP --timestamp "2017-05-25 08:00:00"
-
shardId is one appearing in the describe-stream command above
-
for more info about shard-iterator-type http://docs.aws.amazon.com/kinesis/latest/APIReference/API_GetShardIterator.html#API_GetShardIterator_RequestSyntax
$ aws kinesis get-records --shard-iterator whateversharditeratorwithnumberslettersandslashes --limit 1 --profile user2
{
"Records": [
{
"Data": "some_encoded_data",
"PartitionKey": "some_numbers",
"ApproximateArrivalTimestamp": 1490964900.808,
"SequenceNumber": "some_very_long_number"
}
],
"NextShardIterator": "some_sequence_of_letters_and_numbers_and_slashes",
"MillisBehindLatest": 0
}