fly6516s_Education.test/project-elec
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init code and settings

Browse Source
This commit is contained in:
fly6516 2025-05-11 10:15:41 +00:00
commit a35576502a
279 changed files with 1593 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.idea/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
# Default ignored files
/shelf/
/workspace.xml

112
.idea/artifacts/elec_producer_jar.xml Normal file
View File

@ -0,0 +1,112 @@
<component name="ArtifactManager">
<artifact name="elec-producer:jar">
<output-path>$PROJECT_DIR$/out/artifacts/elec_producer_jar</output-path>
<root id="root">
<element id="archive" name="elec-producer.jar">
<element id="module-output" name="elec-producer" />
<element id="module-output" name="elec-common" />
</element>
<element id="library" level="project" name="Maven: commons-net:commons-net:3.1" />
<element id="library" level="project" name="Maven: com.sun.xml.bind:jaxb-impl:2.2.3-1" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-jobclient:2.5.1" />
<element id="library" level="project" name="Maven: com.sun.jersey.contribs:jersey-guice:1.9" />
<element id="library" level="project" name="Maven: commons-collections:commons-collections:3.2.2" />
<element id="library" level="project" name="Maven: com.sun.jersey:jersey-json:1.9" />
<element id="library" level="project" name="Maven: org.hamcrest:hamcrest-core:1.3" />
<element id="library" level="project" name="Maven: org.apache.httpcomponents:httpcore:4.1.2" />
<element id="library" level="project" name="Maven: org.mortbay.jetty:jsp-api-2.1:6.1.14" />
<element id="library" level="module" name="Maven: jdk.tools:jdk.tools:1.6" module-name="elec-producer" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-client:2.5.1" />
<element id="library" level="project" name="Maven: commons-daemon:commons-daemon:1.0.13" />
<element id="library" level="project" name="Maven: xmlenc:xmlenc:0.52" />
<element id="library" level="project" name="Maven: com.yammer.metrics:metrics-core:2.2.0" />
<element id="library" level="project" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" />
<element id="library" level="project" name="Maven: junit:junit:4.12" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-protocol:1.3.1" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-auth:2.5.1" />
<element id="library" level="project" name="Maven: aopalliance:aopalliance:1.0" />
<element id="library" level="project" name="Maven: javax.xml.bind:jaxb-api:2.2.2" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-hadoop-compat:1.3.1" />
<element id="library" level="project" name="Maven: commons-httpclient:commons-httpclient:3.1" />
<element id="library" level="project" name="Maven: com.jcraft:jsch:0.1.42" />
<element id="library" level="project" name="Maven: com.sun.jersey:jersey-server:1.9" />
<element id="library" level="project" name="Maven: org.mortbay.jetty:jetty:6.1.26" />
<element id="library" level="project" name="Maven: commons-lang:commons-lang:2.6" />
<element id="library" level="project" name="Maven: commons-el:commons-el:1.0" />
<element id="library" level="project" name="Maven: org.codehaus.jackson:jackson-xc:1.8.3" />
<element id="library" level="project" name="Maven: org.codehaus.jettison:jettison:1.1" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-app:2.5.1" />
<element id="library" level="project" name="Maven: com.sun.jersey:jersey-core:1.9" />
<element id="library" level="project" name="Maven: org.apache.htrace:htrace-core:3.1.0-incubating" />
<element id="library" level="project" name="Maven: com.google.inject.extensions:guice-servlet:3.0" />
<element id="library" level="project" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" />
<element id="library" level="project" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" />
<element id="library" level="project" name="Maven: tomcat:jasper-runtime:5.5.23" />
<element id="library" level="project" name="Maven: org.tukaani:xz:1.0" />
<element id="library" level="project" name="Maven: org.mortbay.jetty:jetty-util:6.1.26" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-annotations:2.5.1" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-common:tests:1.3.1" />
<element id="library" level="project" name="Maven: org.slf4j:slf4j-api:1.7.2" />
<element id="library" level="project" name="Maven: javax.inject:javax.inject:1" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-common:1.3.1" />
<element id="library" level="project" name="Maven: org.codehaus.jackson:jackson-core-asl:1.9.13" />
<element id="library" level="project" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" />
<element id="library" level="project" name="Maven: commons-codec:commons-codec:1.9" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-common:2.5.1" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-common:2.5.1" />
<element id="library" level="project" name="Maven: com.github.stephenc.findbugs:findbugs-annotations:1.3.9-1" />
<element id="library" level="project" name="Maven: com.google.inject:guice:3.0" />
<element id="library" level="project" name="Maven: org.codehaus.jackson:jackson-jaxrs:1.9.13" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-yarn-common:2.5.1" />
<element id="library" level="project" name="Maven: com.lmax:disruptor:3.3.0" />
<element id="library" level="project" name="Maven: commons-io:commons-io:2.4" />
<element id="library" level="project" name="Maven: org.apache.commons:commons-math3:3.1.1" />
<element id="library" level="project" name="Maven: com.google.guava:guava:12.0.1" />
<element id="library" level="project" name="Maven: org.apache.commons:commons-math:2.2" />
<element id="library" level="project" name="Maven: org.codehaus.jackson:jackson-mapper-asl:1.9.13" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-annotations:1.3.1" />
<element id="library" level="project" name="Maven: org.apache.commons:commons-compress:1.4.1" />
<element id="library" level="project" name="Maven: asm:asm:3.1" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-core:2.5.1" />
<element id="library" level="project" name="Maven: commons-configuration:commons-configuration:1.6" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-client:1.3.1" />
<element id="library" level="project" name="Maven: org.apache.avro:avro:1.7.4" />
<element id="library" level="project" name="Maven: com.jamesmurty.utils:java-xmlbuilder:0.4" />
<element id="library" level="project" name="Maven: org.jamon:jamon-runtime:2.4.1" />
<element id="library" level="project" name="Maven: com.thoughtworks.paranamer:paranamer:2.3" />
<element id="library" level="project" name="Maven: org.mortbay.jetty:servlet-api-2.5:6.1.14" />
<element id="library" level="project" name="Maven: commons-beanutils:commons-beanutils-core:1.8.0" />
<element id="library" level="project" name="Maven: log4j:log4j:1.2.17" />
<element id="library" level="project" name="Maven: commons-beanutils:commons-beanutils:1.7.0" />
<element id="library" level="project" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" />
<element id="library" level="project" name="Maven: org.jruby.jcodings:jcodings:1.0.8" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-server:1.3.1" />
<element id="library" level="project" name="Maven: commons-digester:commons-digester:1.8" />
<element id="library" level="project" name="Maven: org.mortbay.jetty:jsp-2.1:6.1.14" />
<element id="library" level="project" name="Maven: com.google.protobuf:protobuf-java:2.5.0" />
<element id="library" level="project" name="Maven: commons-cli:commons-cli:1.2" />
<element id="library" level="project" name="Maven: org.slf4j:slf4j-log4j12:1.6.1" />
<element id="library" level="project" name="Maven: tomcat:jasper-compiler:5.5.23" />
<element id="library" level="project" name="Maven: net.java.dev.jets3t:jets3t:0.9.0" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-yarn-api:2.5.1" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-yarn-server-common:2.5.1" />
<element id="library" level="project" name="Maven: com.sun.jersey:jersey-client:1.9" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-prefix-tree:1.3.1" />
<element id="library" level="project" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-procedure:1.3.1" />
<element id="library" level="project" name="Maven: io.netty:netty-all:4.0.23.Final" />
<element id="library" level="project" name="Maven: org.mortbay.jetty:jetty-sslengine:6.1.26" />
<element id="library" level="project" name="Maven: org.jruby.joni:joni:2.1.2" />
<element id="library" level="project" name="Maven: javax.servlet:servlet-api:2.5" />
<element id="library" level="project" name="Maven: javax.activation:activation:1.1" />
<element id="library" level="project" name="Maven: org.fusesource.leveldbjni:leveldbjni-all:1.8" />
<element id="library" level="project" name="Maven: javax.xml.stream:stax-api:1.0-2" />
<element id="library" level="project" name="Maven: org.apache.hbase:hbase-hadoop2-compat:1.3.1" />
<element id="library" level="project" name="Maven: commons-logging:commons-logging:1.2" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-mapreduce-client-shuffle:2.5.1" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-yarn-client:2.5.1" />
<element id="library" level="project" name="Maven: org.xerial.snappy:snappy-java:1.0.4.1" />
<element id="library" level="project" name="Maven: org.apache.hadoop:hadoop-hdfs:2.5.1" />
</root>
</artifact>
</component>

15
.idea/compiler.xml Normal file
View File

@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="elec-producer" />
<module name="elec-common" />
<module name="elec-consumer" />
</profile>
</annotationProcessing>
</component>
</project>

20
.idea/jarRepositories.xml Normal file
View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="RemoteRepositoriesConfiguration">
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Central Repository" />
<option name="url" value="https://repo.maven.apache.org/maven2" />
</remote-repository>
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Maven Central repository" />
<option name="url" value="https://repo1.maven.org/maven2" />
</remote-repository>
<remote-repository>
<option name="id" value="jboss.community" />
<option name="name" value="JBoss Community repository" />
<option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
</remote-repository>
</component>
</project>

14
.idea/misc.xml Normal file
View File

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>

124
.idea/uiDesigner.xml Normal file
View File

@ -0,0 +1,124 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Palette2">
<group name="Swing">
<item class="com.intellij.uiDesigner.HSpacer" tooltip-text="Horizontal Spacer" icon="/com/intellij/uiDesigner/icons/hspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="1" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="com.intellij.uiDesigner.VSpacer" tooltip-text="Vertical Spacer" icon="/com/intellij/uiDesigner/icons/vspacer.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="1" anchor="0" fill="2" />
</item>
<item class="javax.swing.JPanel" icon="/com/intellij/uiDesigner/icons/panel.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3" />
</item>
<item class="javax.swing.JScrollPane" icon="/com/intellij/uiDesigner/icons/scrollPane.png" removable="false" auto-create-binding="false" can-attach-label="true">
<default-constraints vsize-policy="7" hsize-policy="7" anchor="0" fill="3" />
</item>
<item class="javax.swing.JButton" icon="/com/intellij/uiDesigner/icons/button.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="0" fill="1" />
<initial-values>
<property name="text" value="Button" />
</initial-values>
</item>
<item class="javax.swing.JRadioButton" icon="/com/intellij/uiDesigner/icons/radioButton.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="RadioButton" />
</initial-values>
</item>
<item class="javax.swing.JCheckBox" icon="/com/intellij/uiDesigner/icons/checkBox.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="3" anchor="8" fill="0" />
<initial-values>
<property name="text" value="CheckBox" />
</initial-values>
</item>
<item class="javax.swing.JLabel" icon="/com/intellij/uiDesigner/icons/label.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="8" fill="0" />
<initial-values>
<property name="text" value="Label" />
</initial-values>
</item>
<item class="javax.swing.JTextField" icon="/com/intellij/uiDesigner/icons/textField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JPasswordField" icon="/com/intellij/uiDesigner/icons/passwordField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JFormattedTextField" icon="/com/intellij/uiDesigner/icons/formattedTextField.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1">
<preferred-size width="150" height="-1" />
</default-constraints>
</item>
<item class="javax.swing.JTextArea" icon="/com/intellij/uiDesigner/icons/textArea.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTextPane" icon="/com/intellij/uiDesigner/icons/textPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JEditorPane" icon="/com/intellij/uiDesigner/icons/editorPane.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JComboBox" icon="/com/intellij/uiDesigner/icons/comboBox.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="2" anchor="8" fill="1" />
</item>
<item class="javax.swing.JTable" icon="/com/intellij/uiDesigner/icons/table.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JList" icon="/com/intellij/uiDesigner/icons/list.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="2" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTree" icon="/com/intellij/uiDesigner/icons/tree.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3">
<preferred-size width="150" height="50" />
</default-constraints>
</item>
<item class="javax.swing.JTabbedPane" icon="/com/intellij/uiDesigner/icons/tabbedPane.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSplitPane" icon="/com/intellij/uiDesigner/icons/splitPane.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="3" hsize-policy="3" anchor="0" fill="3">
<preferred-size width="200" height="200" />
</default-constraints>
</item>
<item class="javax.swing.JSpinner" icon="/com/intellij/uiDesigner/icons/spinner.png" removable="false" auto-create-binding="true" can-attach-label="true">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSlider" icon="/com/intellij/uiDesigner/icons/slider.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="8" fill="1" />
</item>
<item class="javax.swing.JSeparator" icon="/com/intellij/uiDesigner/icons/separator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="6" anchor="0" fill="3" />
</item>
<item class="javax.swing.JProgressBar" icon="/com/intellij/uiDesigner/icons/progressbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1" />
</item>
<item class="javax.swing.JToolBar" icon="/com/intellij/uiDesigner/icons/toolbar.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="6" anchor="0" fill="1">
<preferred-size width="-1" height="20" />
</default-constraints>
</item>
<item class="javax.swing.JToolBar$Separator" icon="/com/intellij/uiDesigner/icons/toolbarSeparator.png" removable="false" auto-create-binding="false" can-attach-label="false">
<default-constraints vsize-policy="0" hsize-policy="0" anchor="0" fill="1" />
</item>
<item class="javax.swing.JScrollBar" icon="/com/intellij/uiDesigner/icons/scrollbar.png" removable="false" auto-create-binding="true" can-attach-label="false">
<default-constraints vsize-policy="6" hsize-policy="0" anchor="0" fill="2" />
</item>
</group>
</component>
</project>

6
.idea/vcs.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

33
elec-common/pom.xml Normal file
View File

@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>project-elec</artifactId>
<groupId>com.educg</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elec-common</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.3.1</version>
</dependency>
</dependencies>
</project>

16
elec-common/src/main/java/com/educg/elec/common/api/Column.java Normal file
View File

@ -0,0 +1,16 @@
package com.educg.elec.common.api;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
public @interface Column {
String family() default "info";
String column() default "";
}

12
elec-common/src/main/java/com/educg/elec/common/api/RowKey.java Normal file
View File

@ -0,0 +1,12 @@
package com.educg.elec.common.api;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target({ElementType.FIELD})
@Retention(RetentionPolicy.RUNTIME)
public @interface RowKey {
}

12
elec-common/src/main/java/com/educg/elec/common/api/TableRef.java Normal file
View File

@ -0,0 +1,12 @@
package com.educg.elec.common.api;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target({ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface TableRef {
String value();
}

182
elec-common/src/main/java/com/educg/elec/common/bean/BaseDao.java Normal file
View File

@ -0,0 +1,182 @@
package com.educg.elec.common.bean;
import com.educg.elec.common.api.Column;
import com.educg.elec.common.api.RowKey;
import com.educg.elec.common.api.TableRef;
import com.educg.elec.common.constant.Names;
import com.educg.elec.common.constant.ValueConstant;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
public abstract class BaseDao {
private ThreadLocal<Connection> connHolder = new ThreadLocal<Connection>();
private ThreadLocal<Admin> adminHolder = new ThreadLocal<Admin>();
protected void start() throws Exception{
getConnection();
getAdmin();
}
protected void end() throws Exception{
Admin admin = getAdmin();
if(admin != null){
admin.close();
adminHolder.remove();
}
Connection conn = getConnection();
if(conn != null){
conn.close();
connHolder.remove();
}
}
/*
get connection object
*/
protected synchronized Connection getConnection() throws Exception{
Connection conn = connHolder.get();
if(conn == null){
Configuration conf = HBaseConfiguration.create();
conn = ConnectionFactory.createConnection(conf);
connHolder.set(conn);
}
return conn;
}
/*
get admin object
*/
protected synchronized Admin getAdmin() throws Exception{
Admin admin = adminHolder.get();
if(admin == null){
admin = getConnection().getAdmin();
adminHolder.set(admin);
}
return admin;
}
protected void createNamespaceNX(String namespace) throws Exception{
Admin admin = getAdmin();
try {
admin.getNamespaceDescriptor(namespace);
}catch (NamespaceNotFoundException e){
NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create(namespace).build();
admin.createNamespace(namespaceDescriptor);
}
}
protected void createTableXX(String name,String... families) throws Exception{
createTableXX(name,null,families);
}
protected void createTableXX(String name,Integer regionCount,String... families) throws Exception{
Admin admin = getAdmin();
TableName tableName = TableName.valueOf(name);
if(admin.tableExists(tableName)){
//delete table
deleteTable(name);
}
//create table
createTable(name,regionCount,families);
}
protected void deleteTable(String name) throws Exception{
Admin admin = getAdmin();
TableName tableName = TableName.valueOf(name);
admin.disableTable(tableName);
admin.deleteTable(tableName);
}
private void createTable(String name,Integer regionCount,String... families) throws Exception{
Admin admin = getAdmin();
TableName tableName = TableName.valueOf(name);
HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
if(families == null || families.length == 0){
families = new String[1];
families[0] = Names.CF_INFO.getValue();
}
for (String family : families) {
HColumnDescriptor columnDescriptor = new HColumnDescriptor(family);
tableDescriptor.addFamily(columnDescriptor);
}
if(regionCount == null || regionCount <= 1){
admin.createTable(tableDescriptor);
}else {
byte[][] splitKeys = genSplitKeys(regionCount);
admin.createTable(tableDescriptor,splitKeys);
}
}
private byte[][] genSplitKeys(Integer regionCount){
int splitKeyCount = regionCount - 1;
byte[][] bs = new byte[splitKeyCount][];
List<byte[]> bsList = new ArrayList<byte[]>();
for(int i = 0; i < splitKeyCount; i++){
String splitKey = i + "|";
bsList.add(Bytes.toBytes(splitKey));
}
bsList.toArray(bs);
return bs;
}
protected void putData(Object obj) throws Exception{
Class clazz = obj.getClass();
TableRef tableRef = (TableRef) clazz.getAnnotation(TableRef.class);
String tableName = tableRef.value();
String strRowKey = "";
Field[] fs = clazz.getDeclaredFields();
for(Field f : fs){
RowKey rowKey = f.getAnnotation(RowKey.class);
if (rowKey != null){
f.setAccessible(true);
strRowKey = (String)f.get(obj);
break;
}
}
Connection conn = getConnection();
Table table = conn.getTable(TableName.valueOf(tableName));
Put put = new Put(Bytes.toBytes(strRowKey));
for (Field f : fs){
Column column = f.getAnnotation(Column.class);
if (column != null){
String family = column.family();
String colName = column.column();
if ( colName == null || "".equals(colName)){
colName = f.getName();
}
f.setAccessible(true);
String value =(String) f.get(obj);
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(colName), Bytes.toBytes(value));
}
}
//insert
table.put(put);
//close
table.close();
}
protected int genRegionNumber(String date){
int sub = Integer.parseInt(date.substring(0, 4)) - 2006;
int regionNum = sub % ValueConstant.REGION_COUNT;
return regionNum;
}
}

7
elec-common/src/main/java/com/educg/elec/common/bean/Consumer.java Normal file
View File

@ -0,0 +1,7 @@
package com.educg.elec.common.bean;
import java.io.Closeable;
public interface Consumer extends Closeable {
public void consume() throws Exception;
}

11
elec-common/src/main/java/com/educg/elec/common/bean/Val.java Normal file
View File

@ -0,0 +1,11 @@
package com.educg.elec.common.bean;
/**
* value object
*/
public interface Val {
public void setValue(Object val);
public Object getValue();
}

30
elec-common/src/main/java/com/educg/elec/common/constant/Names.java Normal file
View File

@ -0,0 +1,30 @@
package com.educg.elec.common.constant;
import com.educg.elec.common.bean.Val;
/**
* Name Enums
*/
public enum Names implements Val {
NAMESPACE("elec"),
TABLE("elec:eleclog"),
CF_INFO("info"),
TOPIC("eleclog")
;
private String name;
private Names(String name){
this.name=name;
}
@Override
public void setValue(Object val) {
this.name = (String) val;
}
@Override
public String getValue() {
return name;
}
}

5
elec-common/src/main/java/com/educg/elec/common/constant/ValueConstant.java Normal file
View File

@ -0,0 +1,5 @@
package com.educg.elec.common.constant;
public class ValueConstant {
public static final Integer REGION_COUNT = 6;
}

23
elec-common/src/main/java/com/educg/elec/common/util/DateFormatUtil.java Normal file
View File

@ -0,0 +1,23 @@
package com.educg.elec.common.util;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class DateFormatUtil {
public static String format(String originalDate) {
String newDate = null;
SimpleDateFormat originalFormat = new SimpleDateFormat("d/M/yyyy");
originalFormat.setLenient(false);
SimpleDateFormat newFormat = new SimpleDateFormat("yyyyMMdd");
try {
Date date = originalFormat.parse(originalDate);
newDate = newFormat.format(date);
} catch (ParseException e) {
System.out.println("fail: " + originalDate);
e.printStackTrace();
}
return newDate;
}
}

25
elec-common/src/main/java/com/educg/elec/common/util/TimeFormatUtil.java Normal file
View File

@ -0,0 +1,25 @@
package com.educg.elec.common.util;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class TimeFormatUtil {
public static String format(String originalTime){
String newTime = null;
SimpleDateFormat originalFormat = new SimpleDateFormat("HH:mm:ss");
SimpleDateFormat newFormat = new SimpleDateFormat("HHmmss");
try {
Date time = originalFormat.parse(originalTime);
newTime = newFormat.format(time);
System.out.println(newTime);
} catch (ParseException e) {
e.printStackTrace();
}
return newTime;
}
}

BIN
elec-common/target/classes/com/educg/elec/common/api/Column.class Normal file
View File

Binary file not shown.

BIN
elec-common/target/classes/com/educg/elec/common/api/RowKey.class Normal file
View File

Binary file not shown.

BIN
elec-common/target/classes/com/educg/elec/common/api/TableRef.class Normal file
View File

Binary file not shown.

BIN
elec-common/target/classes/com/educg/elec/common/bean/BaseDao.class Normal file
View File

Binary file not shown.

BIN
elec-common/target/classes/com/educg/elec/common/bean/Consumer.class Normal file
View File

Binary file not shown.

BIN
elec-common/target/classes/com/educg/elec/common/bean/Val.class Normal file
View File

Binary file not shown.

BIN
elec-common/target/classes/com/educg/elec/common/constant/Names.class Normal file
View File

Binary file not shown.

BIN
elec-common/target/classes/com/educg/elec/common/constant/ValueConstant.class Normal file
View File

Binary file not shown.

BIN
elec-common/target/classes/com/educg/elec/common/util/DateFormatUtil.class Normal file
View File

Binary file not shown.

BIN
elec-common/target/classes/com/educg/elec/common/util/TimeFormatUtil.class Normal file
View File

Binary file not shown.

39
elec-consumer/pom.xml Normal file
View File

@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>project-elec</artifactId>
<groupId>com.educg</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elec-consumer</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.12</artifactId>
<version>0.11.0.0</version>
</dependency>
<dependency>
<groupId>com.educg</groupId>
<artifactId>elec-common</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

22
elec-consumer/src/main/java/com/educg/elec/consumer/Bootstrap.java Normal file
View File

@ -0,0 +1,22 @@
package com.educg.elec.consumer;
import com.educg.elec.common.bean.Consumer;
import com.educg.elec.consumer.bean.ElecConsumer;
import java.io.IOException;
/**
* Consumer Bootstrap
*/
public class Bootstrap {
public static void main(String[] args) throws Exception {
//create consumer
Consumer consumer = new ElecConsumer();
//consume data
consumer.consume();
//close consumer
consumer.close();
}
}

54
elec-consumer/src/main/java/com/educg/elec/consumer/bean/ElecConsumer.java Normal file
View File

@ -0,0 +1,54 @@
package com.educg.elec.consumer.bean;
import com.educg.elec.common.bean.Consumer;
import com.educg.elec.common.constant.Names;
import com.educg.elec.consumer.dao.HBaseDao;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import java.io.IOException;
import java.time.Duration;
import java.util.Arrays;
import java.util.Properties;
public class ElecConsumer implements Consumer {
@Override
public void consume() throws Exception {
//kafka configuration
Properties prop = new Properties();
try {
prop.load(Thread.currentThread().getContextClassLoader().getResourceAsStream("consumer.properties"));
} catch (IOException e) {
e.printStackTrace();
}
//get flume data
KafkaConsumer<String,String > consumer = new KafkaConsumer<String, String>(prop);
//subscribe kafka topic
consumer.subscribe(Arrays.asList(Names.TOPIC.getValue()));
HBaseDao dao = new HBaseDao();
dao.init();
//consume
Record beforeRecord = new Record();
while (true){
ConsumerRecords<String, String> consumerRecords = consumer.poll(Duration.ofMillis(100));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
// System.out.println(consumerRecord.value());
// dao.insertData(consumerRecord.value());
Record record = new Record(consumerRecord.value());
dao.insertData(record);
System.out.println(record);
}
}
}
@Override
public void close() throws IOException {
}
}

151
elec-consumer/src/main/java/com/educg/elec/consumer/bean/Record.java Normal file
View File

@ -0,0 +1,151 @@
package com.educg.elec.consumer.bean;
import com.educg.elec.common.api.Column;
import com.educg.elec.common.api.RowKey;
import com.educg.elec.common.api.TableRef;
import com.educg.elec.common.util.DateFormatUtil;
import com.educg.elec.common.util.TimeFormatUtil;
/**
* Hbase Record Class
*/
@TableRef("elec:eleclog")
public class Record {
@RowKey
private String rowkey;
@Column
private String date;
@Column
private String time;
@Column
private String globalActivePower;
@Column
private String globalReactivePower;
@Column
private String voltage;
@Column
private String globalIntensity;
@Column
private String subMetering1;
@Column
private String subMetering2;
@Column
private String subMetering3;
public Record(){
}
public Record(String value){
String[] values = value.split("\t");
String date = DateFormatUtil.format(values[0]);
String time = TimeFormatUtil.format(values[1]);
this.date = date;
this.time = time;
this.globalActivePower = values[2];
this.globalReactivePower = values[3];
this.voltage = values[4];
this.globalIntensity = values[5];
this.subMetering1 = values[6];
this.subMetering2 = values[7];
this.subMetering3 = values[8];
}
public String getRowkey() {
return rowkey;
}
public void setRowkey(String rowkey) {
this.rowkey = rowkey;
}
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public String getTime() {
return time;
}
public void setTime(String time) {
this.time = time;
}
public String getGlobalActivePower() {
return globalActivePower;
}
public void setGlobalActivePower(String globalActivePower) {
this.globalActivePower = globalActivePower;
}
public String getGlobalReactivePower() {
return globalReactivePower;
}
public void setGlobalReactivePower(String globalReactivePower) {
this.globalReactivePower = globalReactivePower;
}
public String getVoltage() {
return voltage;
}
public void setVoltage(String voltage) {
this.voltage = voltage;
}
public String getGlobalIntensity() {
return globalIntensity;
}
public void setGlobalIntensity(String globalIntensity) {
this.globalIntensity = globalIntensity;
}
public String getSubMetering1() {
return subMetering1;
}
public void setSubMetering1(String subMetering1) {
this.subMetering1 = subMetering1;
}
public String getSubMetering2() {
return subMetering2;
}
public void setSubMetering2(String subMetering2) {
this.subMetering2 = subMetering2;
}
public String getSubMetering3() {
return subMetering3;
}
public void setSubMetering3(String subMetering3) {
this.subMetering3 = subMetering3;
}
@Override
public String toString() {
return "Record{" +
"rowkey='" + rowkey + '\'' +
", date='" + date + '\'' +
", time='" + time + '\'' +
", globalActivePower='" + globalActivePower + '\'' +
", globalReactivePower='" + globalReactivePower + '\'' +
", voltage='" + voltage + '\'' +
", globalIntensity='" + globalIntensity + '\'' +
", subMetering1='" + subMetering1 + '\'' +
", subMetering2='" + subMetering2 + '\'' +
", subMetering3='" + subMetering3 + '\'' +
'}';
}
}

25
elec-consumer/src/main/java/com/educg/elec/consumer/dao/HBaseDao.java Normal file
View File

@ -0,0 +1,25 @@
package com.educg.elec.consumer.dao;
import com.educg.elec.common.bean.BaseDao;
import com.educg.elec.common.constant.Names;
import com.educg.elec.common.constant.ValueConstant;
import com.educg.elec.consumer.bean.Record;
public class HBaseDao extends BaseDao {
public void init() throws Exception{
start();
createNamespaceNX(Names.NAMESPACE.getValue());
createTableXX(Names.TABLE.getValue(), ValueConstant.REGION_COUNT,Names.CF_INFO.getValue());
end();
}
public void insertData(Record record) throws Exception{
String date = record.getDate();
String time = record.getTime();
String rkDateTime = date+time;
record.setRowkey(genRegionNumber(date)+"_"
+ rkDateTime);
putData(record);
}
}

6
elec-consumer/src/main/resources/consumer.properties Normal file
View File

@ -0,0 +1,6 @@
bootstrap.servers=master:9092
key.deserializer=org.apache.kafka.common.serialization.StringDeserializer
value.deserializer=org.apache.kafka.common.serialization.StringDeserializer
group.id=cg
enable.auto.commit=true
auto.commit.interval.ms=1000

54
elec-consumer/src/main/resources/hbase-site.xml Executable file
View File

@ -0,0 +1,54 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://master:9000/hbase-1.1.5/data</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.zookeeper.property.dataDir</name>
<value>/opt/module/zookeeper-3.4.10/zkData</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>master:2181,slave1:2181,slave2:2181,slave3:2181</value>
</property>
<property>
<name>hbase.security.authorization</name>
<value>true</value>
</property>
<property>
<name>hbase.coprocessor.master.classes</name>
<value>org.apache.hadoop.hbase.security.access.AccessController</value>
</property>
<property>
<name>hbase.coprocessor.region.classes</name>
<value>org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.AccessController</value>
</property>
</configuration>

BIN
elec-consumer/target/classes/com/educg/elec/consumer/Bootstrap.class Normal file
View File

Binary file not shown.

BIN
elec-consumer/target/classes/com/educg/elec/consumer/bean/ElecConsumer.class Normal file
View File

Binary file not shown.

BIN
elec-consumer/target/classes/com/educg/elec/consumer/bean/Record.class Normal file
View File

Binary file not shown.

BIN
elec-consumer/target/classes/com/educg/elec/consumer/dao/HBaseDao.class Normal file
View File

Binary file not shown.

6
elec-consumer/target/classes/consumer.properties Normal file
View File

@ -0,0 +1,6 @@
bootstrap.servers=master:9092
key.deserializer=org.apache.kafka.common.serialization.StringDeserializer
value.deserializer=org.apache.kafka.common.serialization.StringDeserializer
group.id=cg
enable.auto.commit=true
auto.commit.interval.ms=1000

54
elec-consumer/target/classes/hbase-site.xml Executable file
View File

@ -0,0 +1,54 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://master:9000/hbase-1.1.5/data</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.zookeeper.property.dataDir</name>
<value>/opt/module/zookeeper-3.4.10/zkData</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>master:2181,slave1:2181,slave2:2181,slave3:2181</value>
</property>
<property>
<name>hbase.security.authorization</name>
<value>true</value>
</property>
<property>
<name>hbase.coprocessor.master.classes</name>
<value>org.apache.hadoop.hbase.security.access.AccessController</value>
</property>
<property>
<name>hbase.coprocessor.region.classes</name>
<value>org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.AccessController</value>
</property>
</configuration>

27
elec-producer/pom.xml Normal file
View File

@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>project-elec</artifactId>
<groupId>com.educg</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elec-producer</artifactId>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>com.educg</groupId>
<artifactId>elec-common</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

31
elec-producer/src/main/java/com/educg/elec/producer/Bootstrap.java Normal file
View File

@ -0,0 +1,31 @@
package com.educg.elec.producer;
import com.educg.elec.producer.bean.LocalFileProducer;
import com.educg.elec.producer.bean.Producer;
import com.educg.elec.producer.io.LocalFileDataIn;
import com.educg.elec.producer.io.LocalFileDataOut;
/**
* Starter
*/
public class Bootstrap {
public static void main(String[] args) throws Exception{
if(args.length < 2){
System.out.println("invalid arguments");
System.exit(1);
}
String inPath = args[0];
String outPath = args[1];
// create obejct producer
Producer producer = new LocalFileProducer();
producer.setIn(new LocalFileDataIn(inPath));
producer.setOut(new LocalFileDataOut(outPath));
// produce data
producer.produce();
System.out.println("Ending Producer------------------");
// close producer
producer.close();
}
}

19
elec-producer/src/main/java/com/educg/elec/producer/bean/Data.java Normal file
View File

@ -0,0 +1,19 @@
package com.educg.elec.producer.bean;
import com.educg.elec.common.bean.Val;
public abstract class Data implements Val {
public String content;
@Override
public void setValue(Object val) {
content = (String) val;
}
@Override
public String getValue() {
return content;
}
}

16
elec-producer/src/main/java/com/educg/elec/producer/bean/DataIn.java Normal file
View File

@ -0,0 +1,16 @@
package com.educg.elec.producer.bean;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
public interface DataIn extends Closeable {
public Object read() throws IOException;
public <T extends Data> List<T> read(Class<T> clazz) throws IOException;
public void setPath(String path);
}

12
elec-producer/src/main/java/com/educg/elec/producer/bean/DataOut.java Normal file
View File

@ -0,0 +1,12 @@
package com.educg.elec.producer.bean;
import java.io.Closeable;
public interface DataOut extends Closeable {
public void setPath(String path);
public void write( Object data ) throws Exception;
public void write( String data ) throws Exception;
}

109
elec-producer/src/main/java/com/educg/elec/producer/bean/Line.java Normal file
View File

@ -0,0 +1,109 @@
package com.educg.elec.producer.bean;
public class Line extends Data {
private String date;
private String time;
private String globalActivePower;
private String globalReactivePower;
private String voltage;
private String globalIntensity;
private String subMetering1;
private String subMetering2;
private String subMetering3;
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public String getTime() {
return time;
}
public void setTime(String time) {
this.time = time;
}
public String getGlobalActivePower() {
return globalActivePower;
}
public void setGlobalActivePower(String globalActivePower) {
this.globalActivePower = globalActivePower;
}
public String getGlobalReactivePower() {
return globalReactivePower;
}
public void setGlobalReactivePower(String globalReactivePower) {
this.globalReactivePower = globalReactivePower;
}
public String getVoltage() {
return voltage;
}
public void setVoltage(String voltage) {
this.voltage = voltage;
}
public String getGlobalIntensity() {
return globalIntensity;
}
public void setGlobalIntensity(String globalIntensity) {
this.globalIntensity = globalIntensity;
}
public String getSubMetering1() {
return subMetering1;
}
public void setSubMetering1(String subMetering1) {
this.subMetering1 = subMetering1;
}
public String getSubMetering2() {
return subMetering2;
}
public void setSubMetering2(String subMetering2) {
this.subMetering2 = subMetering2;
}
public String getSubMetering3() {
return subMetering3;
}
public void setSubMetering3(String subMetering3) {
this.subMetering3 = subMetering3;
}
@Override
public void setValue(Object val) {
content = (String) val;
String[] values = content.split(";");
setDate(values[0]);
setTime(values[1]);
setGlobalActivePower(values[2]);
setGlobalReactivePower(values[3]);
setVoltage(values[4]);
setGlobalIntensity(values[5]);
setSubMetering1(values[6]);
setSubMetering2(values[7]);
setSubMetering3(values[8]);
}
@Override
public String toString() {
return getDate() + "\t" + getTime() + "\t" +getGlobalActivePower() + "\t" +
getGlobalReactivePower() + "\t" + getVoltage() + "\t" + getGlobalIntensity()
+ "\t" + getSubMetering1() + "\t" + getSubMetering2() + "\t" + getSubMetering3();
}
}

60
elec-producer/src/main/java/com/educg/elec/producer/bean/LocalFileProducer.java Normal file
View File

@ -0,0 +1,60 @@
package com.educg.elec.producer.bean;
import java.io.IOException;
import java.util.List;
/**
* Local File Producer
*/
public class LocalFileProducer implements Producer {
private DataIn in;
private DataOut out;
@Override
public void setIn(DataIn in) {
this.in = in;
}
@Override
public void setOut(DataOut out) {
this.out = out;
}
/**
* produce data
*/
@Override
public void produce() {
try{
List<Line> lines = in.read(Line.class);
int size = lines.size() - 1;
int index = 0;
while(index<=size){
Line curLine = lines.get(index);
out.write(curLine);
Thread.sleep(100);
index++;
}
}
catch (Exception e){
e.printStackTrace();
}
}
/**
* close Producer
* @throws IOException
*/
@Override
public void close() throws IOException {
if(in != null){
in.close();
}
if (out != null) {
out.close();
}
}
}

19
elec-producer/src/main/java/com/educg/elec/producer/bean/Producer.java Normal file
View File

@ -0,0 +1,19 @@
package com.educg.elec.producer.bean;
import java.io.Closeable;
/**
* producer interface
*/
public interface Producer extends Closeable {
public void setIn(DataIn in);
public void setOut(DataOut out);
/**
* produce data
*/
public void produce();
}

68
elec-producer/src/main/java/com/educg/elec/producer/io/LocalFileDataIn.java Normal file
View File

@ -0,0 +1,68 @@
package com.educg.elec.producer.io;
import com.educg.elec.producer.bean.Data;
import com.educg.elec.producer.bean.DataIn;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
/**
* Local File DataIn Obejct
*/
public class LocalFileDataIn implements DataIn {
private BufferedReader reader = null;
public LocalFileDataIn( String path ){
setPath(path);
}
@Override
public Object read() throws IOException {
return null;
}
/**
* read data, return collection
* @param clazz
* @param <T>
* @return
* @throws IOException
*/
@Override
public <T extends Data> List<T> read(Class<T> clazz) throws IOException {
List<T> ts = new ArrayList<T>();
reader.readLine();
try{
String line = null;
while( (line = reader.readLine()) != null){
T t = clazz.newInstance();
t.setValue(line);
ts.add(t);
}
}
catch (Exception e){
e.printStackTrace();
}
return ts;
}
@Override
public void setPath(String path) {
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
@Override
public void close() throws IOException {
if(reader != null) {
reader.close();
}
}
}

52
elec-producer/src/main/java/com/educg/elec/producer/io/LocalFileDataOut.java Normal file
View File

@ -0,0 +1,52 @@
package com.educg.elec.producer.io;
import com.educg.elec.producer.bean.DataOut;
import java.io.*;
/**
* Local File DataOut Object;
*/
public class LocalFileDataOut implements DataOut {
private PrintWriter writer = null;
public LocalFileDataOut( String path ){
setPath(path);
}
@Override
public void setPath(String path) {
try {
writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(path, true), "UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
@Override
public void write(Object data) throws Exception {
this.write(data.toString());
}
/**
* actual write
* @param data
* @throws Exception
*/
@Override
public void write(String data) throws Exception {
System.out.println(data);
writer.println(data);
writer.flush();
}
@Override
public void close() throws IOException {
if (writer != null) {
writer.close();
}
}
}

38
elec-producer/src/main/resources/META-INF/MANIFEST.MF Normal file
View File

@ -0,0 +1,38 @@
Manifest-Version: 1.0
Main-Class: com.educg.elec.producer.Bootstrap
Class-Path: commons-net-3.1.jar jaxb-impl-2.2.3-1.jar hadoop-mapreduce-c
lient-jobclient-2.5.1.jar jersey-guice-1.9.jar commons-collections-3.2.
2.jar jersey-json-1.9.jar hamcrest-core-1.3.jar httpcore-4.1.2.jar jsp-
api-2.1-6.1.14.jar tools.jar hadoop-client-2.5.1.jar commons-daemon-1.0
.13.jar xmlenc-0.52.jar metrics-core-2.2.0.jar apacheds-kerberos-codec-
2.0.0-M15.jar junit-4.12.jar hbase-protocol-1.3.1.jar hadoop-auth-2.5.1
.jar aopalliance-1.0.jar jaxb-api-2.2.2.jar hbase-hadoop-compat-1.3.1.j
ar commons-httpclient-3.1.jar jsch-0.1.42.jar jersey-server-1.9.jar jet
ty-6.1.26.jar commons-lang-2.6.jar commons-el-1.0.jar jackson-xc-1.8.3.
jar jettison-1.1.jar hadoop-mapreduce-client-app-2.5.1.jar jersey-core-
1.9.jar htrace-core-3.1.0-incubating.jar guice-servlet-3.0.jar apacheds
-i18n-2.0.0-M15.jar zookeeper-3.4.6.jar jasper-runtime-5.5.23.jar xz-1.
0.jar jetty-util-6.1.26.jar hadoop-annotations-2.5.1.jar hbase-common-1
.3.1-tests.jar slf4j-api-1.7.2.jar javax.inject-1.jar hbase-common-1.3.
1.jar jackson-core-asl-1.9.13.jar api-util-1.0.0-M20.jar commons-codec-
1.9.jar hadoop-mapreduce-client-common-2.5.1.jar hadoop-common-2.5.1.ja
r findbugs-annotations-1.3.9-1.jar guice-3.0.jar jackson-jaxrs-1.9.13.j
ar hadoop-yarn-common-2.5.1.jar disruptor-3.3.0.jar commons-io-2.4.jar
commons-math3-3.1.1.jar guava-12.0.1.jar commons-math-2.2.jar jackson-m
apper-asl-1.9.13.jar hbase-annotations-1.3.1.jar commons-compress-1.4.1
.jar asm-3.1.jar hadoop-mapreduce-client-core-2.5.1.jar commons-configu
ration-1.6.jar hbase-client-1.3.1.jar avro-1.7.4.jar java-xmlbuilder-0.
4.jar jamon-runtime-2.4.1.jar paranamer-2.3.jar servlet-api-2.5-6.1.14.
jar commons-beanutils-core-1.8.0.jar log4j-1.2.17.jar commons-beanutils
-1.7.0.jar api-asn1-api-1.0.0-M20.jar jcodings-1.0.8.jar hbase-server-1
.3.1.jar commons-digester-1.8.jar jsp-2.1-6.1.14.jar protobuf-java-2.5.
0.jar commons-cli-1.2.jar slf4j-log4j12-1.6.1.jar jasper-compiler-5.5.2
3.jar jets3t-0.9.0.jar hadoop-yarn-api-2.5.1.jar hadoop-yarn-server-com
mon-2.5.1.jar jersey-client-1.9.jar hbase-prefix-tree-1.3.1.jar httpcli
ent-4.2.5.jar hbase-procedure-1.3.1.jar netty-all-4.0.23.Final.jar jett
y-sslengine-6.1.26.jar joni-2.1.2.jar servlet-api-2.5.jar activation-1.
1.jar leveldbjni-all-1.8.jar stax-api-1.0-2.jar hbase-hadoop2-compat-1.
3.1.jar commons-logging-1.2.jar hadoop-mapreduce-client-shuffle-2.5.1.j
ar hadoop-yarn-client-2.5.1.jar snappy-java-1.0.4.1.jar hadoop-hdfs-2.5
.1.jar

38
elec-producer/target/classes/META-INF/MANIFEST.MF Normal file
View File

@ -0,0 +1,38 @@
Manifest-Version: 1.0
Main-Class: com.educg.elec.producer.Bootstrap
Class-Path: commons-net-3.1.jar jaxb-impl-2.2.3-1.jar hadoop-mapreduce-c
lient-jobclient-2.5.1.jar jersey-guice-1.9.jar commons-collections-3.2.
2.jar jersey-json-1.9.jar hamcrest-core-1.3.jar httpcore-4.1.2.jar jsp-
api-2.1-6.1.14.jar tools.jar hadoop-client-2.5.1.jar commons-daemon-1.0
.13.jar xmlenc-0.52.jar metrics-core-2.2.0.jar apacheds-kerberos-codec-
2.0.0-M15.jar junit-4.12.jar hbase-protocol-1.3.1.jar hadoop-auth-2.5.1
.jar aopalliance-1.0.jar jaxb-api-2.2.2.jar hbase-hadoop-compat-1.3.1.j
ar commons-httpclient-3.1.jar jsch-0.1.42.jar jersey-server-1.9.jar jet
ty-6.1.26.jar commons-lang-2.6.jar commons-el-1.0.jar jackson-xc-1.8.3.
jar jettison-1.1.jar hadoop-mapreduce-client-app-2.5.1.jar jersey-core-
1.9.jar htrace-core-3.1.0-incubating.jar guice-servlet-3.0.jar apacheds
-i18n-2.0.0-M15.jar zookeeper-3.4.6.jar jasper-runtime-5.5.23.jar xz-1.
0.jar jetty-util-6.1.26.jar hadoop-annotations-2.5.1.jar hbase-common-1
.3.1-tests.jar slf4j-api-1.7.2.jar javax.inject-1.jar hbase-common-1.3.
1.jar jackson-core-asl-1.9.13.jar api-util-1.0.0-M20.jar commons-codec-
1.9.jar hadoop-mapreduce-client-common-2.5.1.jar hadoop-common-2.5.1.ja
r findbugs-annotations-1.3.9-1.jar guice-3.0.jar jackson-jaxrs-1.9.13.j
ar hadoop-yarn-common-2.5.1.jar disruptor-3.3.0.jar commons-io-2.4.jar
commons-math3-3.1.1.jar guava-12.0.1.jar commons-math-2.2.jar jackson-m
apper-asl-1.9.13.jar hbase-annotations-1.3.1.jar commons-compress-1.4.1
.jar asm-3.1.jar hadoop-mapreduce-client-core-2.5.1.jar commons-configu
ration-1.6.jar hbase-client-1.3.1.jar avro-1.7.4.jar java-xmlbuilder-0.
4.jar jamon-runtime-2.4.1.jar paranamer-2.3.jar servlet-api-2.5-6.1.14.
jar commons-beanutils-core-1.8.0.jar log4j-1.2.17.jar commons-beanutils
-1.7.0.jar api-asn1-api-1.0.0-M20.jar jcodings-1.0.8.jar hbase-server-1
.3.1.jar commons-digester-1.8.jar jsp-2.1-6.1.14.jar protobuf-java-2.5.
0.jar commons-cli-1.2.jar slf4j-log4j12-1.6.1.jar jasper-compiler-5.5.2
3.jar jets3t-0.9.0.jar hadoop-yarn-api-2.5.1.jar hadoop-yarn-server-com
mon-2.5.1.jar jersey-client-1.9.jar hbase-prefix-tree-1.3.1.jar httpcli
ent-4.2.5.jar hbase-procedure-1.3.1.jar netty-all-4.0.23.Final.jar jett
y-sslengine-6.1.26.jar joni-2.1.2.jar servlet-api-2.5.jar activation-1.
1.jar leveldbjni-all-1.8.jar stax-api-1.0-2.jar hbase-hadoop2-compat-1.
3.1.jar commons-logging-1.2.jar hadoop-mapreduce-client-shuffle-2.5.1.j
ar hadoop-yarn-client-2.5.1.jar snappy-java-1.0.4.1.jar hadoop-hdfs-2.5
.1.jar

BIN
elec-producer/target/classes/com/educg/elec/producer/Bootstrap.class Normal file
View File

Binary file not shown.

BIN
elec-producer/target/classes/com/educg/elec/producer/bean/Data.class Normal file
View File

Binary file not shown.

BIN
elec-producer/target/classes/com/educg/elec/producer/bean/DataIn.class Normal file
View File

Binary file not shown.

BIN
elec-producer/target/classes/com/educg/elec/producer/bean/DataOut.class Normal file
View File

Binary file not shown.

BIN
elec-producer/target/classes/com/educg/elec/producer/bean/Line.class Normal file
View File

Binary file not shown.

BIN
elec-producer/target/classes/com/educg/elec/producer/bean/LocalFileProducer.class Normal file
View File

Binary file not shown.

BIN
elec-producer/target/classes/com/educg/elec/producer/bean/Producer.class Normal file
View File

Binary file not shown.

BIN
elec-producer/target/classes/com/educg/elec/producer/io/LocalFileDataIn.class Normal file
View File

Binary file not shown.

BIN
elec-producer/target/classes/com/educg/elec/producer/io/LocalFileDataOut.class Normal file
View File

Binary file not shown.

21
flume-2-kafka.conf Normal file
View File

@ -0,0 +1,21 @@
a1.sources = r1
a1.channels = c1
a1.sinks = k1
a1.sources.r1.type = exec
a1.sources.r1.command = tail -F -c +0 /support/output.txt
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
a1.channels.c1.byteCapacityBufferPercentage = 20
a1.channels.c1.byteCapacity = 800000
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.topic = eleclog
a1.sinks.k1.kafka.bootstrap.servers = master:9092,slave1:9092,slave2:9092,slave3:9092
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

BIN
out/artifacts/elec_producer_jar/activation-1.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/aopalliance-1.0.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/apacheds-i18n-2.0.0-M15.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/apacheds-kerberos-codec-2.0.0-M15.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/api-asn1-api-1.0.0-M20.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/api-util-1.0.0-M20.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/asm-3.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/avro-1.7.4.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-beanutils-1.7.0.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-beanutils-core-1.8.0.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-cli-1.2.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-codec-1.9.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-collections-3.2.2.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-compress-1.4.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-configuration-1.6.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-daemon-1.0.13.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-digester-1.8.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-el-1.0.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-httpclient-3.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-io-2.4.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-lang-2.6.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-logging-1.2.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-math-2.2.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-math3-3.1.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/commons-net-3.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/disruptor-3.3.0.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/elec-producer.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/findbugs-annotations-1.3.9-1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/guava-12.0.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/guice-3.0.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/guice-servlet-3.0.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/hadoop-annotations-2.5.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/hadoop-auth-2.5.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/hadoop-client-2.5.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/hadoop-common-2.5.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/hadoop-hdfs-2.5.1.jar Normal file
View File

Binary file not shown.

BIN
out/artifacts/elec_producer_jar/hadoop-mapreduce-client-app-2.5.1.jar Normal file
View File

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More