Skip to contentSkip to navigationSkip to topbar
Rate this Page:

A Script to Create a Kinesis Stream


You can use the following Bash script to automate the creation of a Kinesis Stream. Copy the code and save it to your computer, for example as create_kinesis_stream.sh.

Run chmod +x create_kinesis_stream.sh to make it executable.

You will also need to install jq, a command line JSON processor on which the script depends. For installation instructions for your OS, please see the jq download page.(link takes you to an external page)

The script also depends upon the AWS CLI, which you will need to install and configure(link takes you to an external page) before executing the script.

The script takes two arguments: your chosen AWS Kinesis Stream name and a shard count.


_110
#!/bin/bash
_110
_110
JQ_CHECK=$(which jq)
_110
if [ -z "$JQ_CHECK" ]; then
_110
echo
_110
echo "This script requires the jq JSON processor. Please install for your OS from https://stedolan.github.io/jq/download/"
_110
echo
_110
exit 1
_110
fi
_110
_110
if [ $# -ne 2 ]; then
_110
echo
_110
echo "usage: $0 <stream_name> <shard_count>"
_110
echo
_110
exit 1
_110
fi
_110
_110
# Set the stream name
_110
STREAM_NAME=${1:-twilio-events}
_110
SHARD_COUNT=${2:-1}
_110
_110
# Create the initial stream
_110
aws kinesis create-stream --stream-name $STREAM_NAME --shard-count $SHARD_COUNT
_110
if [ $? -ne 0 ]; then
_110
echo "Kinesis create failed"
_110
exit 1
_110
fi
_110
_110
# Get the ARN for the Kinesis Stream
_110
KINESIS_ARN=$(aws kinesis describe-stream --stream-name $STREAM_NAME | jq -r .StreamDescription.StreamARN)
_110
_110
# Create the policy for the Kinesis Stream
_110
POLICY_ARN=$(aws iam create-policy --policy-name twilio-events-kinesis-write --policy-document '{
_110
"Version": "2012-10-17",
_110
"Statement": [
_110
{
_110
"Sid": "Quickstart0",
_110
"Effect": "Allow",
_110
"Action": [
_110
"kinesis:PutRecord",
_110
"kinesis:PutRecords"
_110
],
_110
"Resource": "'$KINESIS_ARN'"
_110
},
_110
{
_110
"Sid": "Quickstart1",
_110
"Effect": "Allow",
_110
"Action": [
_110
"kinesis:ListShards",
_110
"kinesis:DescribeLimits"
_110
],
_110
"Resource": "*"
_110
}
_110
]
_110
}' | jq -r .Policy.Arn)
_110
_110
if [ -z "$POLICY_ARN" ]; then
_110
echo "Failed to create IAM policy"
_110
exit 1
_110
fi
_110
_110
# Generate a random external ID
_110
EXTERNAL_ID=$(openssl rand -hex 40)
_110
if [ -z "$EXTERNAL_ID" ]; then
_110
echo "Failed to generate external ID"
_110
exit 1
_110
fi
_110
_110
# This is the Twilio account that needs permissions to be able to assume the role
_110
TWILIO_ASSUME_ROLE_ACCOUNT=${TWILIO_ASSUME_ROLE_ACCOUNT:-arn:aws:iam::177261743968:root}
_110
_110
# Add the random external ID to the the role ARN
_110
# More information can be found here: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user_externalid.html
_110
ROLE_ARN=$(aws iam create-role --role-name twilio-events-kinesis-write --assume-role-policy-document '{
_110
"Version": "2012-10-17",
_110
"Statement": [
_110
{
_110
"Effect": "Allow",
_110
"Principal": {
_110
"AWS": "'$TWILIO_ASSUME_ROLE_ACCOUNT'"
_110
},
_110
"Action": "sts:AssumeRole",
_110
"Condition": {
_110
"StringEquals": {
_110
"sts:ExternalId": "'$EXTERNAL_ID'"
_110
}
_110
}
_110
}
_110
]
_110
}' | jq -r .Role.Arn)
_110
_110
if [ -z "$ROLE_ARN" ]; then
_110
echo "Failed to create IAM role"
_110
exit 1
_110
fi
_110
_110
# Finally attach the policy and the role
_110
aws iam attach-role-policy --role-name twilio-events-kinesis-write --policy-arn $POLICY_ARN
_110
_110
if [ $? -ne 0 ]; then
_110
echo "Attaching policy to role failed"
_110
exit 1
_110
fi
_110
_110
# Print out the values needed for creating the sink in nice JSON
_110
echo "{"
_110
echo '"arn":"'$KINESIS_ARN'",'
_110
echo '"role_arn":"'$ROLE_ARN'",'
_110
echo '"external_id":"'$EXTERNAL_ID'"'
_110
echo "}"


Rate this Page: