Skip to content

Commit 8a77eed

Browse files
author
sboesch
committed
Update to Spark 2.X and add stopwords
1 parent 84d4d5a commit 8a77eed

12 files changed

Lines changed: 942 additions & 185 deletions

File tree

‎pom.xml‎

Lines changed: 108 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -1,106 +1,97 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
12
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2-
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3-
<modelVersion>4.0.0</modelVersion>
4-
<groupId>org.apache.spark.mllib</groupId>
5-
<artifactId>topicModel</artifactId>
6-
<version>1.0-SNAPSHOT</version>
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
<groupId>com.blazedb</groupId>
6+
<artifactId>nlp-hlda</artifactId>
7+
<version>1.0-SNAPSHOT</version>
78

8-
<properties>
9-
<maven.compiler.source>1.6</maven.compiler.source>
10-
<maven.compiler.target>1.6</maven.compiler.target>
11-
<encoding>UTF-8</encoding>
12-
<scala.version>2.10.4</scala.version>
13-
</properties>
9+
<packaging>jar</packaging>
10+
<name>nlp-hlda</name>
1411

15-
<build>
16-
<pluginManagement>
17-
<plugins>
18-
<plugin>
19-
<groupId>net.alchim31.maven</groupId>
20-
<artifactId>scala-maven-plugin</artifactId>
21-
<version>3.1.5</version>
22-
</plugin>
23-
<plugin>
24-
<groupId>org.apache.maven.plugins</groupId>
25-
<artifactId>maven-compiler-plugin</artifactId>
26-
<version>2.0.2</version>
27-
</plugin>
28-
</plugins>
29-
</pluginManagement>
12+
<properties>
13+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
14+
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
3015

31-
<plugins>
32-
33-
<plugin>
34-
<groupId>net.alchim31.maven</groupId>
35-
<artifactId>scala-maven-plugin</artifactId>
36-
<executions>
37-
<execution>
38-
<id>scala-compile-first</id>
39-
<phase>process-resources</phase>
40-
<goals>
41-
<goal>add-source</goal>
42-
<goal>compile</goal>
43-
</goals>
44-
</execution>
45-
<execution>
46-
<id>scala-test-compile</id>
47-
<phase>process-test-resources</phase>
48-
<goals>
49-
<goal>testCompile</goal>
50-
</goals>
51-
</execution>
52-
</executions>
53-
</plugin>
54-
55-
<!-- Plugin to create a single jar that includes all dependencies -->
56-
<plugin>
57-
<artifactId>maven-assembly-plugin</artifactId>
58-
<version>2.4</version>
59-
<configuration>
60-
<descriptorRefs>
61-
<descriptorRef>jar-with-dependencies</descriptorRef>
62-
</descriptorRefs>
63-
</configuration>
64-
<executions>
65-
<execution>
66-
<id>make-assembly</id>
67-
<phase>package</phase>
68-
<goals>
69-
<goal>single</goal>
70-
</goals>
71-
</execution>
72-
</executions>
73-
</plugin>
74-
75-
</plugins>
76-
</build>
16+
<java.version>1.8</java.version>
17+
<spark.version>2.2.1</spark.version>
18+
<scala.binary.version>2.11</scala.binary.version>
19+
<scala.version>2.11.8</scala.version>
20+
<slf4j.version>1.7.5</slf4j.version>
21+
<log4j.version>1.2.17</log4j.version>
22+
</properties>
7723

24+
<repositories>
25+
<repository>
26+
<id>scala-tools.org</id>
27+
<name>Scala-tools Maven2 Repository</name>
28+
<url>http://scala-tools.org/repo-releases</url>
29+
</repository>
30+
<repository>
31+
<id>Akka repository</id>
32+
<url>http://repo.akka.io/releases</url>
33+
</repository>
34+
</repositories>
35+
<pluginRepositories>
36+
<pluginRepository>
37+
<id>scala-tools.org</id>
38+
<name>Scala-tools Maven2 Repository</name>
39+
<url>http://scala-tools.org/repo-releases</url>
40+
</pluginRepository>
41+
</pluginRepositories>
7842
<dependencies>
43+
<dependency>
44+
<groupId>org.apache.spark</groupId>
45+
<artifactId>spark-core_${scala.binary.version}</artifactId>
46+
<version>${spark.version}</version>
47+
</dependency>
48+
<dependency>
49+
<groupId>org.apache.spark</groupId>
50+
<artifactId>spark-mllib_${scala.binary.version}</artifactId>
51+
<version>${spark.version}</version>
52+
</dependency>
53+
<dependency>
54+
<groupId>org.apache.spark</groupId>
55+
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
56+
<version>${spark.version}</version>
57+
</dependency>
58+
59+
<dependency>
60+
<groupId>net.alchim31.maven</groupId>
61+
<artifactId>scala-maven-plugin</artifactId>
62+
<version>3.2.0</version>
63+
</dependency>
7964
<dependency>
8065
<groupId>org.scala-lang</groupId>
8166
<artifactId>scala-library</artifactId>
8267
<version>${scala.version}</version>
8368
</dependency>
84-
85-
<dependency>
86-
<groupId>org.apache.spark</groupId>
87-
<artifactId>spark-core_2.10</artifactId>
88-
<version>1.3.1</version>
89-
</dependency>
90-
<dependency>
69+
<dependency>
70+
<groupId>org.scala-lang</groupId>
71+
<artifactId>scala-compiler</artifactId>
72+
<version>${scala.version}</version>
73+
</dependency>
74+
<dependency>
75+
<groupId>junit</groupId>
76+
<artifactId>junit</artifactId>
77+
<version>4.11</version>
78+
<scope>test</scope>
79+
</dependency>
80+
<dependency>
81+
<groupId>org.scalatest</groupId>
82+
<artifactId>scalatest_${scala.binary.version}</artifactId>
83+
<version>3.0.4</version>
84+
<scope>test</scope>
85+
</dependency>
86+
<dependency>
9187
<groupId>com.github.scopt</groupId>
92-
<artifactId>scopt_2.10</artifactId>
93-
<version>3.2.0</version>
94-
</dependency>
95-
<dependency>
96-
<groupId>org.apache.spark</groupId>
97-
<artifactId>spark-mllib_2.10</artifactId>
98-
<version>1.3.1</version>
99-
</dependency>
100-
<dependency>
88+
<artifactId>scopt_${scala.binary.version}</artifactId>
89+
<version>3.7.0</version>
90+
</dependency>
91+
<dependency>
10192
<groupId>org.scalanlp</groupId>
102-
<artifactId>breeze_2.10</artifactId>
103-
<version>0.11.2</version>
93+
<artifactId>breeze_${scala.binary.version}</artifactId>
94+
<version>0.13.2</version>
10495
<exclusions>
10596
<exclusion>
10697
<groupId>junit</groupId>
@@ -110,8 +101,30 @@
110101
<groupId>org.apache.commons</groupId>
111102
<artifactId>commons-math3</artifactId>
112103
</exclusion>
113-
</exclusions>
114-
</dependency>
115-
116-
</dependencies>
117-
</project>
104+
</exclusions>
105+
</dependency>
106+
</dependencies>
107+
<build>
108+
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
109+
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
110+
<plugins>
111+
<plugin>
112+
<groupId>net.alchim31.maven</groupId>
113+
<artifactId>scala-maven-plugin</artifactId>
114+
<version>3.2.0</version>
115+
<executions>
116+
<execution>
117+
<goals>
118+
<goal>compile</goal>
119+
<goal>testCompile</goal>
120+
</goals>
121+
</execution>
122+
</executions>
123+
</plugin>
124+
<plugin>
125+
<artifactId>maven-compiler-plugin</artifactId>
126+
<version>3.3</version>
127+
</plugin>
128+
</plugins>
129+
</build>
130+
</project>

‎src/main/resources/sample_lda_data.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
1 2 6 0 2 3 1 1 0 0 3
2+
1 3 0 1 3 0 0 2 0 0 1
3+
1 4 1 0 0 4 9 0 1 2 0
4+
2 1 0 3 0 0 5 0 2 3 9
5+
3 1 1 9 3 0 2 0 0 1 3
6+
4 2 0 3 4 5 1 1 1 4 0
7+
2 1 0 3 0 0 5 0 2 2 9
8+
1 1 1 9 2 1 2 0 0 1 3
9+
4 4 0 3 4 2 1 3 0 0 0
10+
2 8 2 0 3 0 2 0 2 7 2
11+
1 1 1 9 0 2 2 0 0 3 3
12+
4 1 0 0 4 5 1 3 0 1 0

0 commit comments

Comments
 (0)