mindol1004/getting-started
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Aws kinesis (#160)

Browse Source 
* added code

* removed cdk code

* added code examples

* added ts files

* added code

Co-authored-by: Pratik Das <pratikd2001@gmail.com>
This commit is contained in:
Pratik Das
2022-03-13 02:14:09 +05:30
committed by GitHub
parent 02c73d52f3
commit b2ab616af8
27 changed files with 1591 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
aws/kinesis/.mvn/wrapper/maven-wrapper.jar vendored Normal file
View File

Binary file not shown.

18
aws/kinesis/.mvn/wrapper/maven-wrapper.properties vendored Normal file
View File

@@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar

8
aws/kinesis/README.md Normal file
View File

@@ -0,0 +1,8 @@
# Getting started with AWS Kinesis
Blog posts about this topic:
* [Getting started with AWS Kinesis](https://reflectoring.io/getting-started-with-aws-kinesis/)

316
aws/kinesis/mvnw vendored Executable file
View File

@@ -0,0 +1,316 @@
#!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Maven Start Up Batch script
#
# Required ENV vars:
# ------------------
# JAVA_HOME - location of a JDK home dir
#
# Optional ENV vars
# -----------------
# M2_HOME - location of maven2's installed home dir
# MAVEN_OPTS - parameters passed to the Java VM when running Maven
# e.g. to debug Maven itself, use
# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
# MAVEN_SKIP_RC - flag to disable loading of mavenrc files
# ----------------------------------------------------------------------------
if [ -z "$MAVEN_SKIP_RC" ] ; then
if [ -f /usr/local/etc/mavenrc ] ; then
. /usr/local/etc/mavenrc
fi
if [ -f /etc/mavenrc ] ; then
. /etc/mavenrc
fi
if [ -f "$HOME/.mavenrc" ] ; then
. "$HOME/.mavenrc"
fi
fi
# OS specific support. $var _must_ be set to either true or false.
cygwin=false;
darwin=false;
mingw=false
case "`uname`" in
CYGWIN*) cygwin=true ;;
MINGW*) mingw=true;;
Darwin*) darwin=true
# Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home
# See https://developer.apple.com/library/mac/qa/qa1170/_index.html
if [ -z "$JAVA_HOME" ]; then
if [ -x "/usr/libexec/java_home" ]; then
export JAVA_HOME="`/usr/libexec/java_home`"
else
export JAVA_HOME="/Library/Java/Home"
fi
fi
;;
esac
if [ -z "$JAVA_HOME" ] ; then
if [ -r /etc/gentoo-release ] ; then
JAVA_HOME=`java-config --jre-home`
fi
fi
if [ -z "$M2_HOME" ] ; then
## resolve links - $0 may be a link to maven's home
PRG="$0"
# need this for relative symlinks
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG="`dirname "$PRG"`/$link"
fi
done
saveddir=`pwd`
M2_HOME=`dirname "$PRG"`/..
# make it fully qualified
M2_HOME=`cd "$M2_HOME" && pwd`
cd "$saveddir"
# echo Using m2 at $M2_HOME
fi
# For Cygwin, ensure paths are in UNIX format before anything is touched
if $cygwin ; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --unix "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
fi
# For Mingw, ensure paths are in UNIX format before anything is touched
if $mingw ; then
[ -n "$M2_HOME" ] &&
M2_HOME="`(cd "$M2_HOME"; pwd)`"
[ -n "$JAVA_HOME" ] &&
JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`"
fi
if [ -z "$JAVA_HOME" ]; then
javaExecutable="`which javac`"
if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then
# readlink(1) is not available as standard on Solaris 10.
readLink=`which readlink`
if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then
if $darwin ; then
javaHome="`dirname \"$javaExecutable\"`"
javaExecutable="`cd \"$javaHome\" && pwd -P`/javac"
else
javaExecutable="`readlink -f \"$javaExecutable\"`"
fi
javaHome="`dirname \"$javaExecutable\"`"
javaHome=`expr "$javaHome" : '\(.*\)/bin'`
JAVA_HOME="$javaHome"
export JAVA_HOME
fi
fi
fi
if [ -z "$JAVACMD" ] ; then
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
else
JAVACMD="`\\unset -f command; \\command -v java`"
fi
fi
if [ ! -x "$JAVACMD" ] ; then
echo "Error: JAVA_HOME is not defined correctly." >&2
echo " We cannot execute $JAVACMD" >&2
exit 1
fi
if [ -z "$JAVA_HOME" ] ; then
echo "Warning: JAVA_HOME environment variable is not set."
fi
CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher
# traverses directory structure from process work directory to filesystem root
# first directory with .mvn subdirectory is considered project base directory
find_maven_basedir() {
if [ -z "$1" ]
then
echo "Path not specified to find_maven_basedir"
return 1
fi
basedir="$1"
wdir="$1"
while [ "$wdir" != '/' ] ; do
if [ -d "$wdir"/.mvn ] ; then
basedir=$wdir
break
fi
# workaround for JBEAP-8937 (on Solaris 10/Sparc)
if [ -d "${wdir}" ]; then
wdir=`cd "$wdir/.."; pwd`
fi
# end of workaround
done
echo "${basedir}"
}
# concatenates all lines of a file
concat_lines() {
if [ -f "$1" ]; then
echo "$(tr -s '\n' ' ' < "$1")"
fi
}
BASE_DIR=`find_maven_basedir "$(pwd)"`
if [ -z "$BASE_DIR" ]; then
exit 1;
fi
##########################################################################################
# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
# This allows using the maven wrapper in projects that prohibit checking in binary data.
##########################################################################################
if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found .mvn/wrapper/maven-wrapper.jar"
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..."
fi
if [ -n "$MVNW_REPOURL" ]; then
jarUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
else
jarUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
fi
while IFS="=" read key value; do
case "$key" in (wrapperUrl) jarUrl="$value"; break ;;
esac
done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties"
if [ "$MVNW_VERBOSE" = true ]; then
echo "Downloading from: $jarUrl"
fi
wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar"
if $cygwin; then
wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"`
fi
if command -v wget > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found wget ... using wget"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
wget "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
else
wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath"
fi
elif command -v curl > /dev/null; then
if [ "$MVNW_VERBOSE" = true ]; then
echo "Found curl ... using curl"
fi
if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then
curl -o "$wrapperJarPath" "$jarUrl" -f
else
curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f
fi
else
if [ "$MVNW_VERBOSE" = true ]; then
echo "Falling back to using Java to download"
fi
javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java"
# For Cygwin, switch paths to Windows format before running javac
if $cygwin; then
javaClass=`cygpath --path --windows "$javaClass"`
fi
if [ -e "$javaClass" ]; then
if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Compiling MavenWrapperDownloader.java ..."
fi
# Compiling the Java class
("$JAVA_HOME/bin/javac" "$javaClass")
fi
if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then
# Running the downloader
if [ "$MVNW_VERBOSE" = true ]; then
echo " - Running MavenWrapperDownloader.java ..."
fi
("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR")
fi
fi
fi
fi
##########################################################################################
# End of extension
##########################################################################################
export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}
if [ "$MVNW_VERBOSE" = true ]; then
echo $MAVEN_PROJECTBASEDIR
fi
MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS"
# For Cygwin, switch paths to Windows format before running java
if $cygwin; then
[ -n "$M2_HOME" ] &&
M2_HOME=`cygpath --path --windows "$M2_HOME"`
[ -n "$JAVA_HOME" ] &&
JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"`
[ -n "$CLASSPATH" ] &&
CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
[ -n "$MAVEN_PROJECTBASEDIR" ] &&
MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"`
fi
# Provide a "standardized" way to retrieve the CLI args that will
# work with both Windows and non-Windows executions.
MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@"
export MAVEN_CMD_LINE_ARGS
WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
exec "$JAVACMD" \
$MAVEN_OPTS \
$MAVEN_DEBUG_OPTS \
-classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \
"-Dmaven.home=${M2_HOME}" \
"-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \
${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@"

188
aws/kinesis/mvnw.cmd vendored Normal file
View File

@@ -0,0 +1,188 @@
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Maven Start Up Batch script
@REM
@REM Required ENV vars:
@REM JAVA_HOME - location of a JDK home dir
@REM
@REM Optional ENV vars
@REM M2_HOME - location of maven2's installed home dir
@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands
@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending
@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven
@REM e.g. to debug Maven itself, use
@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000
@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files
@REM ----------------------------------------------------------------------------
@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on'
@echo off
@REM set title of command window
title %0
@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on'
@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO%
@REM set %HOME% to equivalent of $HOME
if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%")
@REM Execute a user defined script before this one
if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre
@REM check for pre script, once with legacy .bat ending and once with .cmd ending
if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %*
if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %*
:skipRcPre
@setlocal
set ERROR_CODE=0
@REM To isolate internal variables from possible post scripts, we use another setlocal
@setlocal
@REM ==== START VALIDATION ====
if not "%JAVA_HOME%" == "" goto OkJHome
echo.
echo Error: JAVA_HOME not found in your environment. >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
:OkJHome
if exist "%JAVA_HOME%\bin\java.exe" goto init
echo.
echo Error: JAVA_HOME is set to an invalid directory. >&2
echo JAVA_HOME = "%JAVA_HOME%" >&2
echo Please set the JAVA_HOME variable in your environment to match the >&2
echo location of your Java installation. >&2
echo.
goto error
@REM ==== END VALIDATION ====
:init
@REM Find the project base dir, i.e. the directory that contains the folder ".mvn".
@REM Fallback to current working directory if not found.
set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR%
IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir
set EXEC_DIR=%CD%
set WDIR=%EXEC_DIR%
:findBaseDir
IF EXIST "%WDIR%"\.mvn goto baseDirFound
cd ..
IF "%WDIR%"=="%CD%" goto baseDirNotFound
set WDIR=%CD%
goto findBaseDir
:baseDirFound
set MAVEN_PROJECTBASEDIR=%WDIR%
cd "%EXEC_DIR%"
goto endDetectBaseDir
:baseDirNotFound
set MAVEN_PROJECTBASEDIR=%EXEC_DIR%
cd "%EXEC_DIR%"
:endDetectBaseDir
IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig
@setlocal EnableExtensions EnableDelayedExpansion
for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a
@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS%
:endReadAdditionalConfig
SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe"
set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar"
set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO (
IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B
)
@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central
@REM This allows using the maven wrapper in projects that prohibit checking in binary data.
if exist %WRAPPER_JAR% (
if "%MVNW_VERBOSE%" == "true" (
echo Found %WRAPPER_JAR%
)
) else (
if not "%MVNW_REPOURL%" == "" (
SET DOWNLOAD_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar"
)
if "%MVNW_VERBOSE%" == "true" (
echo Couldn't find %WRAPPER_JAR%, downloading it ...
echo Downloading from: %DOWNLOAD_URL%
)
powershell -Command "&{"^
"$webclient = new-object System.Net.WebClient;"^
"if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^
"$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^
"}"^
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^
"}"
if "%MVNW_VERBOSE%" == "true" (
echo Finished downloading %WRAPPER_JAR%
)
)
@REM End of extension
@REM Provide a "standardized" way to retrieve the CLI args that will
@REM work with both Windows and non-Windows executions.
set MAVEN_CMD_LINE_ARGS=%*
%MAVEN_JAVA_EXE% ^
%JVM_CONFIG_MAVEN_PROPS% ^
%MAVEN_OPTS% ^
%MAVEN_DEBUG_OPTS% ^
-classpath %WRAPPER_JAR% ^
"-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^
%WRAPPER_LAUNCHER% %MAVEN_CONFIG% %*
if ERRORLEVEL 1 goto error
goto end
:error
set ERROR_CODE=1
:end
@endlocal & set ERROR_CODE=%ERROR_CODE%
if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost
@REM check for post script, once with legacy .bat ending and once with .cmd ending
if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat"
if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd"
:skipRcPost
@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on'
if "%MAVEN_BATCH_PAUSE%"=="on" pause
if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE%
cmd /C exit /B %ERROR_CODE%

72
aws/kinesis/pom.xml Normal file
View File

@@ -0,0 +1,72 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>io.pratik</groupId>
<artifactId>kinesisexamples</artifactId>
<version>0.0.1-SNAPSHOT</version>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>kinesis</artifactId>
</dependency>
<!-- <dependency> <groupId>com.amazonaws</groupId> <artifactId>amazon-kinesis-producer</artifactId>
<version>0.14.10</version> </dependency> <dependency> <groupId>software.amazon.kinesis</groupId>
<artifactId>amazon-kinesis-client</artifactId> <version>2.3.10</version>
</dependency> -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>firehose</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>kinesisanalyticsv2</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>kinesisvideo</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>kinesisvideomedia</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>amazon-kinesis-video-streams-producer-sdk-java</artifactId>
<version>1.12.0</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.11</artifactId>
<version>1.14.3</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.11</artifactId>
<version>1.14.3</version>
</dependency>
<dependency>
<groupId>software.amazon.kinesis</groupId>
<artifactId>amazon-kinesis-connector-flink</artifactId>
<version>2.3.0</version>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>2.17.116</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
</project>

16
aws/kinesis/src/main/java/io/pratik/Constants.java Normal file
View File

@@ -0,0 +1,16 @@
/**
*
*/
package io.pratik;
import software.amazon.awssdk.regions.Region;
/**
* @author pratikdas
*
*/
public interface Constants {
String MY_DATA_STREAM = "mydatastream";
String AWS_PROFILE_NAME = "tompoc";
Region AWS_REGION = Region.AP_SOUTHEAST_2;
}

79
aws/kinesis/src/main/java/io/pratik/DataStreamResourceHelper.java Normal file
View File

@@ -0,0 +1,79 @@
/**
*
*/
package io.pratik;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.kinesis.KinesisClient;
import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest;
import software.amazon.awssdk.services.kinesis.model.CreateStreamResponse;
import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryRequest;
import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryResponse;
import software.amazon.awssdk.services.kinesis.model.ResourceNotFoundException;
import software.amazon.awssdk.services.kinesis.model.StreamDescriptionSummary;
import software.amazon.awssdk.services.kinesis.model.StreamMode;
import software.amazon.awssdk.services.kinesis.model.StreamModeDetails;
import software.amazon.awssdk.services.kinesis.model.StreamStatus;
/**
* @author pratikdas
*
*/
public class DataStreamResourceHelper {
/**
* @param args
*/
public static void main(String[] args) {
createDataStream();
}
public static void createDataStream() {
KinesisClient kinesisClient = getKinesisClient();
CreateStreamRequest createStreamRequest = CreateStreamRequest.builder().streamName(Constants.MY_DATA_STREAM).streamModeDetails(StreamModeDetails.builder().streamMode(StreamMode.ON_DEMAND).build()).build();
CreateStreamResponse createStreamResponse = kinesisClient.createStream(createStreamRequest);
DescribeStreamSummaryRequest describeStreamSummaryRequest = DescribeStreamSummaryRequest.builder().streamName(Constants.MY_DATA_STREAM ).build();
DescribeStreamSummaryResponse describeStreamSummaryResponse = kinesisClient.describeStreamSummary(describeStreamSummaryRequest );
long startTime = System.currentTimeMillis();
long endTime = startTime + ( 10 * 60 * 1000 );
while ( System.currentTimeMillis() < endTime ) {
try {
Thread.sleep(20 * 1000);
}
catch ( Exception e ) {}
try {
StreamDescriptionSummary streamDescSumm = describeStreamSummaryResponse.streamDescriptionSummary();
if(streamDescSumm.streamStatus().equals(StreamStatus.ACTIVE)) break;
try {
Thread.sleep( 1000 );
}catch ( Exception e ) {}
}catch ( ResourceNotFoundException e ) {}
}
}
private static KinesisClient getKinesisClient() {
AwsCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME);
KinesisClient kinesisClient = KinesisClient
.builder()
.credentialsProvider(credentialsProvider)
.region(Region.US_EAST_1).build();
return kinesisClient;
}
}

79
aws/kinesis/src/main/java/io/pratik/EventConsumer.java Normal file
View File

@@ -0,0 +1,79 @@
/**
*
*/
package io.pratik;
import java.util.List;
import java.util.logging.Logger;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.kinesis.KinesisClient;
import software.amazon.awssdk.services.kinesis.model.GetRecordsRequest;
import software.amazon.awssdk.services.kinesis.model.GetRecordsResponse;
import software.amazon.awssdk.services.kinesis.model.GetShardIteratorRequest;
import software.amazon.awssdk.services.kinesis.model.GetShardIteratorResponse;
import software.amazon.awssdk.services.kinesis.model.Record;
import software.amazon.awssdk.services.kinesis.model.ShardIteratorType;
/**
* @author pratikdas
*
*/
public class EventConsumer {
private static final Logger logger = Logger.getLogger(EventConsumer.class.getName());
/**
* @param args
*/
public static void main(String[] args) {
receiveEvents();
}
public static void receiveEventsWithKCL() {}
public static void receiveEvents() {
KinesisClient kinesisClient = getKinesisClient();
String shardId = "shardId-000000000001";
GetShardIteratorRequest getShardIteratorRequest = GetShardIteratorRequest.builder().streamName(Constants.MY_DATA_STREAM).shardId(shardId).shardIteratorType(ShardIteratorType.TRIM_HORIZON.name()).build();
GetShardIteratorResponse getShardIteratorResponse = kinesisClient.getShardIterator(getShardIteratorRequest );
String shardIterator = getShardIteratorResponse.shardIterator();
logger.info("shardIterator " + shardIterator);
while(shardIterator != null) {
GetRecordsRequest getRecordsRequest = GetRecordsRequest.builder().shardIterator(shardIterator).limit(5).build();
GetRecordsResponse getRecordsResponse = kinesisClient.getRecords(getRecordsRequest );
List<Record> records = getRecordsResponse.records();
logger.info("count "+records.size());
records.forEach(record->{
byte[] dataInBytes = record.data().asByteArray();
logger.info(new String(dataInBytes));
});
shardIterator = getRecordsResponse.nextShardIterator();
}
kinesisClient.close();
}
private static KinesisClient getKinesisClient() {
AwsCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME);
KinesisClient kinesisClient = KinesisClient
.builder()
.credentialsProvider(credentialsProvider)
.region(Region.US_EAST_1)
.build();
return kinesisClient;
}
}

104
aws/kinesis/src/main/java/io/pratik/EventSender.java Normal file
View File

@@ -0,0 +1,104 @@
/**
*
*/
package io.pratik;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.kinesis.KinesisClient;
import software.amazon.awssdk.services.kinesis.model.PutRecordRequest;
import software.amazon.awssdk.services.kinesis.model.PutRecordResponse;
import software.amazon.awssdk.services.kinesis.model.PutRecordsRequest;
import software.amazon.awssdk.services.kinesis.model.PutRecordsRequestEntry;
import software.amazon.awssdk.services.kinesis.model.PutRecordsResponse;
/**
* @author pratikdas
*
*/
public class EventSender {
private static final Logger logger = Logger.getLogger(EventSender.class.getName());
/**
* @param args
*/
public static void main(String[] args) {
sendEvent();
}
public static void sendEvent() {
KinesisClient kinesisClient = getKinesisClient();
String partitionKey = String.format("partitionKey-%d", 1);
SdkBytes data = SdkBytes.fromByteBuffer(ByteBuffer.wrap("Test data".getBytes()));
PutRecordRequest putRecordRequest
= PutRecordRequest
.builder()
.streamName(Constants.MY_DATA_STREAM)
.partitionKey(partitionKey)
.data(data)
.build();
PutRecordResponse putRecordResult
= kinesisClient.putRecord(putRecordRequest);
logger.info("Put Result" + putRecordResult);
kinesisClient.close();
}
public static void sendEvents() {
KinesisClient kinesisClient = getKinesisClient();
String partitionKey = String.format("partitionKey-%d", 1);
List <PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
for (int i = 0; i < 5; i++) {
SdkBytes data = SdkBytes
.fromByteBuffer(ByteBuffer.wrap(("Test event "+i).getBytes()));
PutRecordsRequestEntry putRecordsRequestEntry
= PutRecordsRequestEntry.builder()
.data(data)
.partitionKey(partitionKey)
.build();
putRecordsRequestEntryList.add(putRecordsRequestEntry);
}
PutRecordsRequest putRecordsRequest
= PutRecordsRequest
.builder()
.streamName(Constants.MY_DATA_STREAM)
.records(putRecordsRequestEntryList)
.build();
PutRecordsResponse putRecordsResult = kinesisClient
.putRecords(putRecordsRequest);
logger.info("Put Result" + putRecordsResult);
kinesisClient.close();
}
private static KinesisClient getKinesisClient() {
AwsCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME);
KinesisClient kinesisClient = KinesisClient
.builder()
.credentialsProvider(credentialsProvider)
.region(Region.US_EAST_1).build();
return kinesisClient;
}
}

63
aws/kinesis/src/main/java/io/pratik/FirehoseEventSender.java Normal file
View File

@@ -0,0 +1,63 @@
/**
*
*/
package io.pratik;
import java.nio.ByteBuffer;
import java.util.logging.Logger;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.firehose.FirehoseClient;
import software.amazon.awssdk.services.firehose.model.PutRecordRequest;
import software.amazon.awssdk.services.firehose.model.PutRecordResponse;
import software.amazon.awssdk.services.firehose.model.Record;
/**
* @author pratikdas
*
*/
public class FirehoseEventSender {
private final static Logger logger = Logger.getLogger(FirehoseEventSender.class.getName());
/**
* @param args
*/
public static void main(String[] args) {
new FirehoseEventSender().sendEvent();
}
public void sendEvent() {
String deliveryStreamName= "PUT-S3-5ZGgA";
String data = "Test data" + "\n";
;
Record record = Record.builder().data(SdkBytes.fromByteArray(data.getBytes())).build();
PutRecordRequest putRecordRequest = PutRecordRequest
.builder()
.deliveryStreamName(deliveryStreamName).record(record).build();
FirehoseClient firehoseClient = getFirehoseClient();
// Put record into the DeliveryStream
PutRecordResponse putRecordResponse = firehoseClient.putRecord(putRecordRequest);
logger.info("record ID:: " + putRecordResponse.recordId());
firehoseClient.close();
}
private static FirehoseClient getFirehoseClient() {
AwsCredentialsProvider credentialsProvider = ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME);
FirehoseClient kinesisClient = FirehoseClient
.builder()
.credentialsProvider(credentialsProvider)
.region(Constants.AWS_REGION).build();
return kinesisClient;
}
}

84
aws/kinesis/src/main/java/io/pratik/FirehoseResourceHelper.java Normal file
View File

@@ -0,0 +1,84 @@
/**
*
*/
package io.pratik;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.firehose.FirehoseClient;
import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamRequest;
import software.amazon.awssdk.services.firehose.model.CreateDeliveryStreamResponse;
import software.amazon.awssdk.services.firehose.model.DeliveryStreamType;
import software.amazon.awssdk.services.firehose.model.ExtendedS3DestinationConfiguration;
import software.amazon.awssdk.services.firehose.model.KinesisStreamSourceConfiguration;
import software.amazon.awssdk.services.firehose.model.S3DestinationConfiguration;
import software.amazon.awssdk.services.kinesis.KinesisClient;
import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest;
import software.amazon.awssdk.services.kinesis.model.CreateStreamResponse;
import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryRequest;
import software.amazon.awssdk.services.kinesis.model.DescribeStreamSummaryResponse;
import software.amazon.awssdk.services.kinesis.model.ResourceNotFoundException;
import software.amazon.awssdk.services.kinesis.model.StreamDescriptionSummary;
import software.amazon.awssdk.services.kinesis.model.StreamMode;
import software.amazon.awssdk.services.kinesis.model.StreamModeDetails;
import software.amazon.awssdk.services.kinesis.model.StreamStatus;
/**
* @author pratikdas
*
*/
public class FirehoseResourceHelper {
/**
* @param args
*/
public static void main(String[] args) {
createDeliveryStream();
}
public static void createDeliveryStream() {
FirehoseClient firehoseClient = getFirehoseClient();
String kinesisStreamARN = "";
String roleARN = "";
KinesisStreamSourceConfiguration kinesisStreamSourceConfiguration =
KinesisStreamSourceConfiguration.builder().build();
String bucketARN = "";
ExtendedS3DestinationConfiguration s3DestinationConfiguration
= ExtendedS3DestinationConfiguration.builder()
.bucketARN(bucketARN)
.build();
String streamName = "";
CreateDeliveryStreamRequest createDeliveryStreamRequest =
CreateDeliveryStreamRequest
.builder()
.deliveryStreamName(streamName )
.deliveryStreamType(DeliveryStreamType.DIRECT_PUT)
.kinesisStreamSourceConfiguration(kinesisStreamSourceConfiguration )
.extendedS3DestinationConfiguration(s3DestinationConfiguration )
.build();
CreateDeliveryStreamResponse response = firehoseClient.createDeliveryStream(createDeliveryStreamRequest );
firehoseClient.close();
}
private static FirehoseClient getFirehoseClient() {
AwsCredentialsProvider credentialsProvider =
ProfileCredentialsProvider.create(Constants.AWS_PROFILE_NAME);
FirehoseClient firehoseClient =
FirehoseClient.builder()
.credentialsProvider(credentialsProvider)
.region(Region.US_EAST_1)
.build();
return firehoseClient;
}
}

64
aws/kinesis/src/main/java/io/pratik/StreamingDataGenerator.java Normal file
View File

@@ -0,0 +1,64 @@
/**
*
*/
package io.pratik;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.URL;
/**
* @author pratikdas
*
*/
public class StreamingDataGenerator {
/**
* @param args
*/
public static void main(String[] args) {
try {
new StreamingDataGenerator().generate();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private void generate() throws Exception {
File file = getFile("apache_access_log");
// Note: Double backquote is to avoid compiler
// interpret words
// like \test as \t (ie. as a escape sequence)
// Creating an object of BufferedReader class
BufferedReader br
= new BufferedReader(new FileReader(file));
// Declaring a string variable
String st;
// Condition holds true till
// there is character in a string
while ((st = br.readLine()) != null) {
System.out.println(st);
break;
}
}
private File getFile(String fileName) throws IOException
{
ClassLoader classLoader = getClass().getClassLoader();
URL resource = classLoader.getResource(fileName);
if (resource == null) {
throw new IllegalArgumentException("file is not found!");
} else {
return new File(resource.getFile());
}
}
}

120
aws/kinesis/src/main/java/io/pratik/flink/ErrorCounter.java Normal file
View File

@@ -0,0 +1,120 @@
package io.pratik.flink;
import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.util.Properties;
import java.util.logging.Logger;
import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
import io.pratik.Constants;
import io.pratik.models.LogRecord;
import software.amazon.kinesis.connectors.flink.FlinkKinesisProducer;
import software.amazon.kinesis.connectors.flink.config.ConsumerConfigConstants;
public class ErrorCounter {
private final static Logger logger = Logger.getLogger(ErrorCounter.class.getName());
private static final String FILE_PATH = "/Users/pratikdas/eclipse-workspace/kinesisexamples/src/main/resources/apache_access_log";
public static void main(String[] args) throws Exception {
// set up the streaming execution environment
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
DataStream<String> inputStream = createSource(env);
DataStream<LogRecord> logRecords = inputStream.flatMap(new FlatMapFunction<String, LogRecord>() {
@Override
public void flatMap(String value, Collector<LogRecord> out) throws Exception {
String[] parts = value.split("\\s+");
LogRecord record = new LogRecord();
record.setIp(parts[0]);
record.setHttpStatus(parts[8]);
record.setUrl(parts[6]);
out.collect(record);
}
});
DataStream<LogRecord> errorRecords = logRecords.filter(new FilterFunction<LogRecord>() {
@Override
public boolean filter(LogRecord value) throws Exception {
boolean matched = !value.getHttpStatus().equalsIgnoreCase("200");
return matched;
}
});
DataStream<String> keyedStream = errorRecords.keyBy(value -> value.getIp()).flatMap(new FlatMapFunction<LogRecord, String>() {
@Override
public void flatMap(LogRecord value, Collector<String> out) throws Exception {
out.collect(value.getUrl()+"::" + value.getHttpStatus());
}
});
//TODO Uncomment this code for deploying to Kinesis Data Analytics
// keyedStream.addSink(createSink());
keyedStream.print();
env.execute("Error alerts");
}
/*private static void createSink(final StreamExecutionEnvironment env, DataStream<LogRecord> input) {
input.print();
}*/
private static DataStream<String> createSource(final StreamExecutionEnvironment env) {
return env.readTextFile(
FILE_PATH);
}
//TODO Uncomment this code for deploying to Kinesis Data Analytics
/*private static DataStream<String> createSource(final StreamExecutionEnvironment env) {
Properties inputProperties = new Properties();
inputProperties.setProperty(ConsumerConfigConstants.AWS_REGION, Constants.AWS_REGION.toString());
inputProperties.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "LATEST");
String inputStreamName = "in-app-log-stream";
return env.addSource(new FlinkKinesisConsumer<>(inputStreamName, new SimpleStringSchema(), inputProperties));
}*/
private static FlinkKinesisProducer<String> createSink() {
Properties outputProperties = new Properties();
outputProperties.setProperty(ConsumerConfigConstants.AWS_REGION, Constants.AWS_REGION.toString());
FlinkKinesisProducer<String> sink = new FlinkKinesisProducer<>(new SimpleStringSchema(), outputProperties);
String outputStreamName = "log_data_stream";
sink.setDefaultStream(outputStreamName);
sink.setDefaultPartition("0");
return sink;
}
private File getFile(String fileName) throws IOException {
ClassLoader classLoader = getClass().getClassLoader();
URL resource = classLoader.getResource(fileName);
if (resource == null) {
throw new IllegalArgumentException("file is not found!");
} else {
return new File(resource.getFile());
}
}
}

40
aws/kinesis/src/main/java/io/pratik/models/LogRecord.java Normal file
View File

@@ -0,0 +1,40 @@
/**
*
*/
package io.pratik.models;
/**
* @author pratikdas
*
*/
public class LogRecord {
private String ip;
private String httpStatus;
private String url;
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public String getHttpStatus() {
return httpStatus;
}
public void setHttpStatus(String httpStatus) {
this.httpStatus = httpStatus;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
@Override
public String toString() {
return "LogRecord [ip=" + ip + ", httpStatus=" + httpStatus + ", url=" + url + "]";
}
}

33
aws/kinesis/src/main/resources/apache_access_log Normal file
View File

@@ -0,0 +1,33 @@
83.149.9.216 - - [17/May/2015:10:05:03 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
83.149.9.216 - - [17/May/2015:10:05:43 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-dashboard3.png HTTP/1.1" 200 171717 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
83.149.9.216 - - [17/May/2015:10:05:47 +0000] "GET /presentations/logstash-monitorama-2013/plugin/highlight/highlight.js HTTP/1.1" 200 26185 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
83.149.9.216 - - [17/May/2015:10:05:12 +0000] "GET /presentations/logstash-monitorama-2013/plugin/zoom-js/zoom.js HTTP/1.1" 200 7697 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
83.149.9.216 - - [17/May/2015:10:05:07 +0000] "GET /presentations/logstash-monitorama-2013/plugin/notes/notes.js HTTP/1.1" 200 2892 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
83.149.9.216 - - [17/May/2015:10:05:34 +0000] "GET /presentations/logstash-monitorama-2013/images/sad-medic.png HTTP/1.1" 200 430406 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
83.149.9.216 - - [17/May/2015:10:05:57 +0000] "GET /presentations/logstash-monitorama-2013/css/fonts/Roboto-Bold.ttf HTTP/1.1" 404 38720 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
83.149.9.216 - - [17/May/2015:10:05:50 +0000] "GET /presentations/logstash-monitorama-2013/css/fonts/Roboto-Regular.ttf HTTP/1.1" 200 41820 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
83.149.9.216 - - [17/May/2015:10:05:24 +0000] "GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1" 310 52878 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"
46.105.14.53 - - [17/May/2015:10:05:44 +0000] "GET /blog/tags/puppet?flav=rss20 HTTP/1.1" 200 14872 "-" "UniversalFeedParser/4.2-pre-314-svn +http://feedparser.org/"
20.241.23.22 - - [17/May/2015:10:05:47 +0000] "GET /blog/geekery/soekris-gpio.html HTTP/1.0" 200 9587 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)"
91.177.205.119 - - [17/May/2015:10:05:22 +0000] "GET /blog/geekery/xvfb-firefox.html HTTP/1.1" 200 10975 "http://en.wikipedia.org/wiki/Xvfb" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"
91.177.205.119 - - [17/May/2015:10:05:34 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"
91.177.205.119 - - [17/May/2015:10:05:37 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"
91.177.205.119 - - [17/May/2015:10:05:54 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"
91.177.205.119 - - [17/May/2015:10:05:31 +0000] "GET /images/web/2009/banner.png HTTP/1.1" 200 52315 "http://semicomplete.com/blog/geekery/xvfb-firefox.html" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)"
91.177.205.119 - - [17/May/2015:10:05:32 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "-" "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Win64; x64; Trident/6.0)"
66.249.73.185 - - [17/May/2015:10:05:22 +0000] "GET /doc/index.html?org/elasticsearch/action/search/SearchResponse.html HTTP/1.1" 404 294 "-" "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
207.241.237.228 - - [17/May/2015:10:05:40 +0000] "GET /blog/tags/defcon HTTP/1.0" 200 24142 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)"
207.241.237.101 - - [17/May/2015:10:05:51 +0000] "GET /blog/tags/regex HTTP/1.0" 200 14888 "http://www.semicomplete.com/blog/tags/C" "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)"
87.169.99.232 - - [17/May/2015:10:05:59 +0000] "GET /presentations/puppet-at-loggly/puppet-at-loggly.pdf.html HTTP/1.1" 200 24747 "https://www.google.de/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36"
209.85.238.199 - - [17/May/2015:10:05:30 +0000] "GET /blog/tags/firefox?flav=rss20 HTTP/1.1" 200 16021 "-" "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 3 subscribers; feed-id=14171215010336145331)"
209.85.238.199 - - [17/May/2015:10:05:15 +0000] "GET /test.xml HTTP/1.1" 200 1370 "-" "Feedfetcher-Google; (+http://www.google.com/feedfetcher.html; 1 subscribers; feed-id=11390274670024826467)"
81.220.24.207 - - [17/May/2015:10:05:13 +0000] "GET /blog/geekery/ssl-latency.html HTTP/1.1" 200 17147 "http://www.google.fr/url?sa=t&rct=j&q=&esrc=s&source=web&cd=5&ved=0CE4QFjAE&url=http%3A%2F%2Fwww.semicomplete.com%2Fblog%2Fgeekery%2Fssl-latency.html&ei=ZdEAU9mGGuWX1AW09IDoBw&usg=AFQjCNHw6zioJpizqX8Q0YpKKaF4zdCSEg&bvm=bv.61535280,d.d2k" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11"
81.220.24.207 - - [17/May/2015:10:05:44 +0000] "GET /reset.css HTTP/1.1" 200 1015 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11"
81.220.24.207 - - [17/May/2015:10:05:26 +0000] "GET /images/jordan-80.png HTTP/1.1" 200 6146 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11"
81.220.24.207 - - [17/May/2015:10:05:39 +0000] "GET /style2.css HTTP/1.1" 200 4877 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11"
81.220.24.207 - - [17/May/2015:10:05:52 +0000] "GET /images/web/2009/banner.png HTTP/1.1" 200 52315 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11"
81.220.24.207 - - [17/May/2015:10:05:21 +0000] "GET /favicon.ico HTTP/1.1" 200 3638 "http://www.semicomplete.com/blog/geekery/ssl-latency.html" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.73.11 (KHTML, like Gecko) Version/7.0.1 Safari/537.73.11"
66.249.73.135 - - [17/May/2015:11:05:17 +0000] "GET /blog/geekery/vmware-cpu-performance.html HTTP/1.1" 200 12908 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
46.105.14.53 - - [17/May/2015:11:05:42 +0000] "GET /blog/tags/puppet?flav=rss20 HTTP/1.1" 200 14872 "-" "UniversalFeedParser/4.2-pre-314-svn +http://feedparser.org/"
218.30.103.62 - - [17/May/2015:11:05:11 +0000] "GET /robots.txt HTTP/1.1" 200 - "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"
218.30.103.62 - - [17/May/2015:11:05:46 +0000] "GET /robots.txt HTTP/1.1" 200 - "-" "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)"

1
build-all.sh
View File

@@ -87,6 +87,7 @@ if [[ "$MODULE" == "module7" ]]
then then
# ADD NEW MODULES HERE # ADD NEW MODULES HERE
# (add new modules above the rest so you get quicker feedback if it fails) # (add new modules above the rest so you get quicker feedback if it fails)
build maven_module "aws/kinesis"
build maven_module "aws/sqs" build maven_module "aws/sqs"
build_maven_module "core-java/annotation-processing/introduction-to-annotations" build_maven_module "core-java/annotation-processing/introduction-to-annotations"

3
node/express/getting-started/README.md Normal file
View File

@@ -0,0 +1,3 @@
# Related Blog Posts
* [Getting Started with Express](https://reflectoring.io/getting-started-with-express/)

26
node/express/getting-started/app.ts Normal file
View File

@@ -0,0 +1,26 @@
import express, { Request, Response, NextFunction } from 'express';
const app = express();
const port = 3000;
interface Product {
name: string;
price: number;
brand: string;
};
const getProducts = (request: Request, response: Response, next: NextFunction) => {
let products: Product[] = [
{"name":"television", "price":112.34, "brand":"samsung"},
{"name":"washing machine", "price": 345.34, "brand": "LG"},
{"name":"Macbook", "price": 3454.34, "brand": "Apple"}
]
response.status(200).json(products);
}
app.get('/products', getProducts);
app.listen(port, () => {
console.log(`Server listening at port ${port}.`);
});

19
node/express/getting-started/errormiddleware.js Normal file
View File

@@ -0,0 +1,19 @@
// errormiddleware.js
const errorLogger = (err, request, response, next) => {
console.log( `error ${err.message}`)
next(err) // calling next middleware
}
const errorResponder = (err, request, response, next) => {
response.header("Content-Type", 'application/json')
const status = err.status || 400
response.status(status).send(err.message)
}
const invalidPathHandler = (request, response, next) => {
response.status(400)
response.send('invalid path')
}
module.exports = { errorLogger, errorResponder, invalidPathHandler }

102
node/express/getting-started/index.js Normal file
View File

@@ -0,0 +1,102 @@
const express = require('express');
const app = express();
const requestLogger = (request, response, next) => {
console.log(request);
next();
};
const requireJsonContent = (request, response, next) => {
if (request.headers['content-type'] !== 'application/json') {
response.status(400).send('Server requires application/json')
} else {
next()
}
}
app.use(requestLogger);
app.use((err, request, response, next) => {
response.status(500).send('Error!')
})
// Set the body size limit to 100 bytes
app.use(express.json({ limit: 100 }));
app.use(express.urlencoded({ extended: false }));
let products = [{"name":"television", "price":112.34, "brand":"samsung"},
{"name":"washing machine", "price": 345.34, "brand": "LG"},
{"name":"Macbook", "price": 3454.34, "brand": "Apple"}];
// handle get request for path /
app.get('/', (request, response) => {
response.send('response for GET request');
});
// handle get request for path /products
app.get('/products', (request, response) => {
response.json(products);
});
// handle get request for path /products
app.get('/products/:brand', (request, response) => {
const brand = req.params.brand;
console.log("brand ${brand} " + brand);
const productsFiltered = products.filter(product=> product.brand == brand);
response.json(productsFiltered);
});
// handle post request for path /products
app.post('/products', requireJsonContent, (request, response) => {
const products = [];
const name = req.body.name ;
const brand = req.body.brand;
console.log(name + " " + brand);
products.push({name: req.body.name, brand: req.body.brand, price: req.body.price});
const productCreationResponse = {productID: "12345", result: "success"};
response.json(productCreationResponse);
});
app.set('view engine', 'pug')
app.set('views', './views')
app.get('/home', (request, response) => {
response.render("home", { title: "Home", message: "My home page" , sysdate: new Date().toLocaleString()})
})
// handle get request with 3 middleware functions
app.get('/users', (request, response, next) => {
console.log("Stage 1 processing ");
next()
},
(request, response, next) => {
console.log("Stage 2 processing ");
next();
},
(request, response) => {
response.send('response for GET request');
});
app.get('/products/error', (request, response) => {
throw new Error("processing error!")
});
// start the server
const port = 3000
app.listen(3000,
() => console.log(`Server listening on port ${port}.`));

16
node/express/getting-started/middleware.js Normal file
View File

@@ -0,0 +1,16 @@
// middleware.js
const requestLogger = (request, response, next) => {
console.log(request);
next();
};
const requireJsonContent = (request, response, next) => {
if (request.headers['content-type'] !== 'application/json') {
response.status(400).send('Server requires application/json')
} else {
next()
}
}
module.exports = { requestLogger, requireJsonContent}

24
node/express/getting-started/package.json Normal file
View File

@@ -0,0 +1,24 @@
{
"name": "storefront",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"serve": "nodemon app.ts"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"express": "^4.17.2",
"pug": "^3.0.2"
},
"devDependencies": {
"@types/express": "^4.17.13",
"@types/node": "^17.0.17",
"nodemon": "^2.0.15",
"ts-node": "^10.5.0",
"typescript": "^4.5.5"
}
}

75
node/express/getting-started/routes.js Normal file
View File

@@ -0,0 +1,75 @@
// routes.js
const express = require('express')
const { requireJsonContent } = require('./middleware')
const router = express.Router()
// Set the body size limit to 100 bytes
router.use(express.json({ limit: 100 }))
let products = [{"name":"television", "price":112.34, "brand":"samsung"},
{"name":"washing machine", "price": 345.34, "brand": "LG"},
{"name":"Macbook", "price": 3454.34, "brand": "Apple"}];
// handle get request for path /products
router.get('/products', (request, response) => {
console.log(products)
response.json(products)
});
// handle get request for path /products
router.get('/products/:brand', (request, response) => {
const brand = request.params.brand;
console.log(`brand ${brand} `);
const productsFiltered = products.filter(product=> product.brand == brand);
response.json(productsFiltered);
});
// handle post request for path /products
router.post('/products', requireJsonContent,(request, response) => {
const name = request.body.name ;
const brand = request.body.brand;
console.log(name + " " + brand);
products.push({name: request.body.name, brand: request.body.brand, price: request.body.price});
const productCreationResponse = {productID: "12345", result: "success"};
response.json(productCreationResponse);
});
// handle put request for path /products
router.put('/products', (request, response) => {
const name = request.body.name ;
const brand = request.body.brand;
console.log(name + " " + brand);
products.push({name: request.body.name, brand: request.body.brand, price: request.body.price});
const productCreationResponse = {productID: "12345", result: "success"};
response.json(productCreationResponse);
})
router.get('/home', (request, response) => {
console.log("rendering...")
response.render("home", { title: "Home", message: "My home page" , sysdate: new Date().toLocaleString()})
})
router.get('/productswitherror', (request, response) => {
let err = new Error("processing error ")
err.statusCode = 400
throw err
});
module.exports = router

25
node/express/getting-started/server.js Normal file
View File

@@ -0,0 +1,25 @@
// server.js
const express = require('express')
const routes = require('./routes')
const { requestLogger } = require('./middleware')
const { errorLogger, errorResponder, invalidPathHandler } = require('./errormiddleware')
const app = express()
const PORT = process.env.PORT || 3000
app.use(requestLogger)
app.use(routes)
app.use(errorLogger)
app.use(errorResponder)
app.use(invalidPathHandler)
app.set('view engine', 'pug')
app.set('views', './views')
app.listen(PORT, () => {
console.log(`Server listening at http://localhost:${PORT}`)
})

8
node/express/getting-started/tsconfig.json Normal file
View File

@@ -0,0 +1,8 @@
{
"compilerOptions": {
"module": "commonjs",
"target": "es6",
"rootDir": "./",
"esModuleInterop": true
}
}

8
node/express/getting-started/views/home.pug Normal file
View File

@@ -0,0 +1,8 @@
html
head
title= title
body
h1= message
div
p Generated by express at
span= sysdate