parent
6eec0ae436
commit
4cb14eab2c
|
|
@ -12,16 +12,16 @@
|
|||
http://www.springframework.org/schema/tx
|
||||
http://www.springframework.org/schema/tx/spring-tx-3.0.xsd
|
||||
http://www.springframework.org/schema/aop
|
||||
http://www.springframework.org/schema/aop/spring-aop-3.0.xsd">
|
||||
|
||||
http://www.springframework.org/schema/aop/spring-aop-3.0.xsd">
|
||||
|
||||
<!--SourceData -->
|
||||
<bean id="dataSourceOne" class="org.apache.commons.dbcp.BasicDataSource"
|
||||
destroy-method="close">
|
||||
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
|
||||
<property name="url"
|
||||
value="jdbc:mysql://localhost/ossean_production?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="gather" />
|
||||
<property name="password" value="Influx@1234" />
|
||||
value="jdbc:mysql://localhost:3306/ossean_production?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="root" />
|
||||
<property name="password" value="NUDTpdl@123" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
<property name="testOnBorrow" value="true"/>
|
||||
</bean>
|
||||
|
|
@ -29,18 +29,19 @@
|
|||
<property name="dataSource" ref="dataSourceOne" />
|
||||
</bean>
|
||||
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
|
||||
<property name="basePackage" value="com.ossean.dao" />
|
||||
<property name="basePackage" value="com.ossean" />
|
||||
<property name="sqlSessionFactory" ref="sqlSessionFactoryOne"></property>
|
||||
</bean>
|
||||
|
||||
|
||||
<bean id="dataSourceTwo" class="org.apache.commons.dbcp.BasicDataSource"
|
||||
<!--DestData -->
|
||||
<!-- <bean id="dataSourceTwo" class="org.apache.commons.dbcp.BasicDataSource"
|
||||
destroy-method="close">
|
||||
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
|
||||
<property name="url"
|
||||
value="jdbc:mysql://172.16.128.29:3306/extract_result?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="gather" />
|
||||
<property name="password" value="influx1234" />
|
||||
value="jdbc:mysql://192.168.80.130:3306/ossean_production?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="trustie" />
|
||||
<property name="password" value="1234" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
<property name="testOnBorrow" value="true"/>
|
||||
</bean>
|
||||
|
|
@ -48,9 +49,9 @@
|
|||
<property name="dataSource" ref="dataSourceTwo" />
|
||||
</bean>
|
||||
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
|
||||
<property name="basePackage" value="com.ossean.dao2" />
|
||||
<property name="basePackage" value="com.ossean.databaseDest" />
|
||||
<property name="sqlSessionFactory" ref="sqlSessionFactoryTwo"></property>
|
||||
</bean>
|
||||
</bean> -->
|
||||
|
||||
<tx:annotation-driven transaction-manager="transactionManager"/>
|
||||
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
|
||||
|
|
|
|||
|
|
@ -1,20 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
find ./target/classes -name "*.properties"|xargs rm -f
|
||||
find ./target/classes -name "*.xml"|xargs rm -f
|
||||
find ./target/classes -name "*.dic"|xargs rm -f
|
||||
|
||||
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
|
||||
|
||||
tmp='./target/classes':$tmp
|
||||
tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
||||
tmp='./bin/resources':$tmp
|
||||
CLASSPATH=$tmp:$CLASSPATH
|
||||
|
||||
|
||||
echo $CLASSPATH
|
||||
JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
||||
#echo JVM_ARGS=$JVM_ARGS
|
||||
#ulimit -n 400000
|
||||
#echo "" > nohup.out
|
||||
java $JVM_ARGS -classpath $CLASSPATH com.ossean.MergeProjects >>log/merge_projects_2017.log 2>&1 &
|
||||
sh bin/start_merge_projects_1.sh
|
||||
sh bin/start_merge_projects_2.sh
|
||||
sh bin/start_merge_projects_3.sh
|
||||
sh bin/start_merge_projects_4.sh
|
||||
sh bin/start_merge_projects_others.sh
|
||||
|
|
@ -17,4 +17,4 @@ JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxT
|
|||
#echo JVM_ARGS=$JVM_ARGS
|
||||
#ulimit -n 400000
|
||||
#echo "" > nohup.out
|
||||
java $JVM_ARGS -classpath $CLASSPATH com.ossean.GetSynonyms >>log/getsynonyms_2017.log 2>&1 &
|
||||
java $JVM_ARGS -classpath $CLASSPATH com.ossean.MergeProjects >>log/merge_projects_1027.log 2>&1 &
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
find ./target/classes -name "*.properties"|xargs rm -f
|
||||
find ./target/classes -name "*.xml"|xargs rm -f
|
||||
find ./target/classes -name "*.dic"|xargs rm -f
|
||||
|
||||
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
|
||||
|
||||
tmp='./target/classes':$tmp
|
||||
tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
||||
tmp='./bin/resources':$tmp
|
||||
CLASSPATH=$tmp:$CLASSPATH
|
||||
|
||||
|
||||
echo $CLASSPATH
|
||||
JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
||||
#echo JVM_ARGS=$JVM_ARGS
|
||||
#ulimit -n 400000
|
||||
#echo "" > nohup.out
|
||||
java $JVM_ARGS -classpath $CLASSPATH com.ossean.MergeProjects edd_relations_pointer2 >>log/merge_projects200001-400000.log 2>&1 &
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
find ./target/classes -name "*.properties"|xargs rm -f
|
||||
find ./target/classes -name "*.xml"|xargs rm -f
|
||||
find ./target/classes -name "*.dic"|xargs rm -f
|
||||
|
||||
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
|
||||
|
||||
tmp='./target/classes':$tmp
|
||||
tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
||||
tmp='./bin/resources':$tmp
|
||||
CLASSPATH=$tmp:$CLASSPATH
|
||||
|
||||
|
||||
echo $CLASSPATH
|
||||
JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
||||
#echo JVM_ARGS=$JVM_ARGS
|
||||
#ulimit -n 400000
|
||||
#echo "" > nohup.out
|
||||
java $JVM_ARGS -classpath $CLASSPATH com.ossean.MergeProjects edd_relations_pointer3 >>log/merge_projects400001-600000.log 2>&1 &
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
find ./target/classes -name "*.properties"|xargs rm -f
|
||||
find ./target/classes -name "*.xml"|xargs rm -f
|
||||
find ./target/classes -name "*.dic"|xargs rm -f
|
||||
|
||||
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
|
||||
|
||||
tmp='./target/classes':$tmp
|
||||
tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
||||
tmp='./bin/resources':$tmp
|
||||
CLASSPATH=$tmp:$CLASSPATH
|
||||
|
||||
|
||||
echo $CLASSPATH
|
||||
JVM_ARGS="-Xmn98m -Xmx1024m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
||||
#echo JVM_ARGS=$JVM_ARGS
|
||||
#ulimit -n 400000
|
||||
#echo "" > nohup.out
|
||||
java $JVM_ARGS -classpath $CLASSPATH com.ossean.MergeProjects edd_relations_pointer4 >>log/merge_projects600001-800000.log 2>&1 &
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
|
||||
find ./target/classes -name "*.properties"|xargs rm -f
|
||||
find ./target/classes -name "*.xml"|xargs rm -f
|
||||
find ./target/classes -name "*.dic"|xargs rm -f
|
||||
|
||||
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
|
||||
|
||||
tmp='./target/classes':$tmp
|
||||
tmp='./target/Project_Match-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
||||
tmp='./bin/resources':$tmp
|
||||
CLASSPATH=$tmp:$CLASSPATH
|
||||
|
||||
|
||||
echo $CLASSPATH
|
||||
JVM_ARGS="-Xmn98m -Xmx512m -Xms512m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
||||
#echo JVM_ARGS=$JVM_ARGS
|
||||
#ulimit -n 400000
|
||||
#echo "" > nohup.out
|
||||
java $JVM_ARGS -classpath $CLASSPATH com.ossean.MergeProjectsByHomepageOnly >>log/MergeProjectsByHomepageOnly.log 2>&1 &
|
||||
|
|
@ -11,18 +11,4 @@ javascript
|
|||
sql
|
||||
asp
|
||||
php
|
||||
perl
|
||||
swift
|
||||
angularjs
|
||||
angular2
|
||||
angular
|
||||
node.js
|
||||
jqury
|
||||
html
|
||||
spark
|
||||
nginx
|
||||
spring
|
||||
kde
|
||||
vim
|
||||
twitter
|
||||
opencv
|
||||
perl
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,520 +0,0 @@
|
|||
17-01-19 18:25:36,978 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 18:25:37,971 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 0.01655 minutes
|
||||
17-01-19 18:26:24,129 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:1000 projects cost: 0.7841 minutes
|
||||
17-01-19 18:26:53,234 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:2000 projects cost: 1.2691833 minutes
|
||||
17-01-19 18:27:15,255 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:3000 projects cost: 1.6362 minutes
|
||||
17-01-19 18:27:34,267 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:4000 projects cost: 1.95305 minutes
|
||||
17-01-19 18:27:51,615 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:5000 projects cost: 2.2422 minutes
|
||||
17-01-19 18:28:07,927 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:6000 projects cost: 2.5140667 minutes
|
||||
17-01-19 18:28:23,346 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:7000 projects cost: 2.77105 minutes
|
||||
17-01-19 18:28:39,365 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:8000 projects cost: 3.0380332 minutes
|
||||
17-01-19 18:28:53,549 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:9000 projects cost: 3.2744334 minutes
|
||||
17-01-19 18:29:07,074 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:10000 projects cost: 3.49985 minutes
|
||||
17-01-19 18:29:20,829 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:11000 projects cost: 3.7291 minutes
|
||||
17-01-19 18:29:34,769 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:12000 projects cost: 3.9614334 minutes
|
||||
17-01-19 18:29:47,627 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:13000 projects cost: 4.1757336 minutes
|
||||
17-01-19 18:30:01,112 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:14000 projects cost: 4.400483 minutes
|
||||
17-01-19 18:30:14,901 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:15000 projects cost: 4.6303 minutes
|
||||
17-01-19 18:30:28,075 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:16000 projects cost: 4.849867 minutes
|
||||
17-01-19 18:30:37,972 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 18:30:37,972 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 18:30:38,759 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 5.029683 minutes
|
||||
17-01-19 18:30:41,266 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:17000 projects cost: 5.0697165 minutes
|
||||
17-01-19 18:30:54,618 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:18000 projects cost: 5.29225 minutes
|
||||
17-01-19 18:31:07,702 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:19000 projects cost: 5.510317 minutes
|
||||
17-01-19 18:31:20,825 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:20000 projects cost: 5.7290335 minutes
|
||||
17-01-19 18:31:33,718 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:21000 projects cost: 5.943917 minutes
|
||||
17-01-19 18:31:46,811 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:22000 projects cost: 6.162133 minutes
|
||||
17-01-19 18:32:00,371 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:23000 projects cost: 6.3881335 minutes
|
||||
17-01-19 18:32:13,192 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:24000 projects cost: 6.6018167 minutes
|
||||
17-01-19 18:32:26,246 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:25000 projects cost: 6.819383 minutes
|
||||
17-01-19 18:32:38,921 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:26000 projects cost: 7.0306334 minutes
|
||||
17-01-19 18:32:50,899 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:27000 projects cost: 7.2302666 minutes
|
||||
17-01-19 18:33:02,752 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:28000 projects cost: 7.427817 minutes
|
||||
17-01-19 18:33:15,815 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:29000 projects cost: 7.6455336 minutes
|
||||
17-01-19 18:33:29,566 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:30000 projects cost: 7.8747168 minutes
|
||||
17-01-19 18:33:41,817 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:31000 projects cost: 8.0789 minutes
|
||||
17-01-19 18:33:55,188 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:32000 projects cost: 8.30175 minutes
|
||||
17-01-19 18:34:11,091 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:33000 projects cost: 8.5668 minutes
|
||||
17-01-19 18:34:25,755 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:34000 projects cost: 8.8112 minutes
|
||||
17-01-19 18:34:39,636 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:35000 projects cost: 9.04255 minutes
|
||||
17-01-19 18:34:54,047 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:36000 projects cost: 9.282733 minutes
|
||||
17-01-19 18:35:08,139 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:37000 projects cost: 9.5176 minutes
|
||||
17-01-19 18:35:22,236 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:38000 projects cost: 9.75255 minutes
|
||||
17-01-19 18:35:38,205 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:39000 projects cost: 10.018683 minutes
|
||||
17-01-19 18:35:38,759 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 18:35:38,759 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 18:35:39,683 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 10.045083 minutes
|
||||
17-01-19 18:35:51,569 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:40000 projects cost: 10.241433 minutes
|
||||
17-01-19 18:36:05,212 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:41000 projects cost: 10.468817 minutes
|
||||
17-01-19 18:36:17,296 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:42000 projects cost: 10.670217 minutes
|
||||
17-01-19 18:36:29,828 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:43000 projects cost: 10.879084 minutes
|
||||
17-01-19 18:36:42,350 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:44000 projects cost: 11.087767 minutes
|
||||
17-01-19 18:36:54,629 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:45000 projects cost: 11.292417 minutes
|
||||
17-01-19 18:37:08,888 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:46000 projects cost: 11.530084 minutes
|
||||
17-01-19 18:37:21,984 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:47000 projects cost: 11.74835 minutes
|
||||
17-01-19 18:37:35,202 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:48000 projects cost: 11.96865 minutes
|
||||
17-01-19 18:37:48,590 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:49000 projects cost: 12.191783 minutes
|
||||
17-01-19 18:38:01,139 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:50000 projects cost: 12.400933 minutes
|
||||
17-01-19 18:38:13,536 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:51000 projects cost: 12.60755 minutes
|
||||
17-01-19 18:38:25,733 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:52000 projects cost: 12.810833 minutes
|
||||
17-01-19 18:38:38,101 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:53000 projects cost: 13.016967 minutes
|
||||
17-01-19 18:38:50,695 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:54000 projects cost: 13.226867 minutes
|
||||
17-01-19 18:39:04,231 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:55000 projects cost: 13.452467 minutes
|
||||
17-01-19 18:39:17,938 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:56000 projects cost: 13.680917 minutes
|
||||
17-01-19 18:39:31,237 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:57000 projects cost: 13.902567 minutes
|
||||
17-01-19 18:39:44,649 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:58000 projects cost: 14.1261 minutes
|
||||
17-01-19 18:39:58,142 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:59000 projects cost: 14.350984 minutes
|
||||
17-01-19 18:40:12,516 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:60000 projects cost: 14.59055 minutes
|
||||
17-01-19 18:40:27,094 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:61000 projects cost: 14.833517 minutes
|
||||
17-01-19 18:40:39,684 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 18:40:39,684 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 18:40:40,537 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 15.059317 minutes
|
||||
17-01-19 18:40:42,420 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:62000 projects cost: 15.08895 minutes
|
||||
17-01-19 18:40:55,219 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:63000 projects cost: 15.302267 minutes
|
||||
17-01-19 18:41:07,633 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:64000 projects cost: 15.509167 minutes
|
||||
17-01-19 18:41:19,950 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:65000 projects cost: 15.71445 minutes
|
||||
17-01-19 18:41:32,150 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:66000 projects cost: 15.917784 minutes
|
||||
17-01-19 18:41:45,665 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:67000 projects cost: 16.143034 minutes
|
||||
17-01-19 18:41:59,416 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:68000 projects cost: 16.372217 minutes
|
||||
17-01-19 18:42:13,380 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:69000 projects cost: 16.60495 minutes
|
||||
17-01-19 18:42:31,089 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:70000 projects cost: 16.9001 minutes
|
||||
17-01-19 18:42:45,969 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:71000 projects cost: 17.1481 minutes
|
||||
17-01-19 18:43:00,428 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:72000 projects cost: 17.389084 minutes
|
||||
17-01-19 18:43:15,015 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:73000 projects cost: 17.6322 minutes
|
||||
17-01-19 18:43:29,640 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:74000 projects cost: 17.87595 minutes
|
||||
17-01-19 18:43:46,572 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:75000 projects cost: 18.15815 minutes
|
||||
17-01-19 18:44:02,693 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:76000 projects cost: 18.426834 minutes
|
||||
17-01-19 18:44:16,689 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:77000 projects cost: 18.6601 minutes
|
||||
17-01-19 18:44:30,666 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:78000 projects cost: 18.89305 minutes
|
||||
17-01-19 18:44:43,424 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:79000 projects cost: 19.105684 minutes
|
||||
17-01-19 18:44:56,102 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:80000 projects cost: 19.316967 minutes
|
||||
17-01-19 18:45:09,026 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:81000 projects cost: 19.532383 minutes
|
||||
17-01-19 18:45:22,090 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:82000 projects cost: 19.750116 minutes
|
||||
17-01-19 18:45:35,353 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:83000 projects cost: 19.971167 minutes
|
||||
17-01-19 18:45:40,538 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 18:45:40,538 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 18:45:41,393 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 20.073584 minutes
|
||||
17-01-19 18:45:48,544 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:84000 projects cost: 20.191017 minutes
|
||||
17-01-19 18:46:01,547 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:85000 projects cost: 20.407734 minutes
|
||||
17-01-19 18:46:14,565 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:86000 projects cost: 20.6247 minutes
|
||||
17-01-19 18:46:27,647 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:87000 projects cost: 20.842733 minutes
|
||||
17-01-19 18:46:40,134 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:88000 projects cost: 21.05085 minutes
|
||||
17-01-19 18:46:52,281 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:89000 projects cost: 21.2533 minutes
|
||||
17-01-19 18:47:00,935 WARN com.ossean.MergeProjects(MergeProjects.java:80) ## deal with:90000 projects cost: 21.397533 minutes
|
||||
17-01-19 18:47:08,576 WARN com.ossean.MergeProjects(MergeProjects.java:63) ## deal with 90844 relaitons cost: 21.524883 minutes
|
||||
17-01-19 20:50:21,040 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 20:50:22,021 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 0.01635 minutes
|
||||
17-01-19 20:55:22,021 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 20:55:22,021 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 20:55:22,763 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 5.0287166 minutes
|
||||
17-01-19 21:00:22,763 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 21:00:22,763 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 21:00:23,456 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 10.040267 minutes
|
||||
17-01-19 21:05:23,456 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 21:05:23,456 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 21:05:24,184 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 15.0524 minutes
|
||||
17-01-19 21:09:32,303 WARN com.ossean.MergeProjects(MergeProjects.java:63) ## deal with 183095 projects cost: 19.1861 minutes
|
||||
17-01-19 21:53:56,752 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 21:53:57,439 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 0.01145 minutes
|
||||
17-01-19 21:56:26,187 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 21:56:26,854 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 0.011116667 minutes
|
||||
17-01-19 21:57:30,952 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 21:57:31,630 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 0.0113 minutes
|
||||
17-01-19 21:58:27,729 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 21:58:34,354 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.10118333 minutes
|
||||
17-01-19 21:58:34,355 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 21:58:39,304 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.078 minutes
|
||||
17-01-19 21:58:39,304 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 21:58:43,258 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.063766666 minutes
|
||||
17-01-19 21:58:43,258 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 21:58:47,080 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.061633334 minutes
|
||||
17-01-19 21:58:47,080 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:01:41,413 WARN com.ossean.GetSynonyms(GetSynonyms.java:51) ## ==============start insert synonyms============
|
||||
17-01-19 22:01:42,534 WARN com.ossean.MergeProjects(MergeProjects.java:62) ## deal with 0 projects cost: 0.016716667 minutes
|
||||
17-01-19 22:06:13,913 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:21,184 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.111666664 minutes
|
||||
17-01-19 22:06:21,184 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:26,585 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.08575 minutes
|
||||
17-01-19 22:06:26,585 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:30,862 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.068783335 minutes
|
||||
17-01-19 22:06:30,862 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:34,981 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.06628333 minutes
|
||||
17-01-19 22:06:34,981 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:39,421 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.072233334 minutes
|
||||
17-01-19 22:06:39,421 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:43,066 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.059616666 minutes
|
||||
17-01-19 22:06:43,066 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:46,341 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.053466666 minutes
|
||||
17-01-19 22:06:46,341 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:49,998 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.059233334 minutes
|
||||
17-01-19 22:06:49,998 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:53,289 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.053483333 minutes
|
||||
17-01-19 22:06:53,289 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:06:56,627 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.05415 minutes
|
||||
17-01-19 22:06:56,627 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:00,127 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.05685 minutes
|
||||
17-01-19 22:07:00,127 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:03,480 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.054583333 minutes
|
||||
17-01-19 22:07:03,481 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:06,926 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.055633333 minutes
|
||||
17-01-19 22:07:06,926 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:10,382 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.055816665 minutes
|
||||
17-01-19 22:07:10,382 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:13,522 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.050783332 minutes
|
||||
17-01-19 22:07:13,522 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:16,540 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04865 minutes
|
||||
17-01-19 22:07:16,540 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:19,702 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.051266667 minutes
|
||||
17-01-19 22:07:19,702 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:22,628 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.047183335 minutes
|
||||
17-01-19 22:07:22,628 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:25,547 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.047233332 minutes
|
||||
17-01-19 22:07:25,547 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:28,425 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0463 minutes
|
||||
17-01-19 22:07:28,425 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:31,307 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.046466667 minutes
|
||||
17-01-19 22:07:31,307 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:34,229 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.047133334 minutes
|
||||
17-01-19 22:07:34,229 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:37,235 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.048416667 minutes
|
||||
17-01-19 22:07:37,235 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:40,271 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04885 minutes
|
||||
17-01-19 22:07:40,271 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:43,538 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0527 minutes
|
||||
17-01-19 22:07:43,538 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:46,365 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.045333333 minutes
|
||||
17-01-19 22:07:46,365 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:49,469 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.049916666 minutes
|
||||
17-01-19 22:07:49,469 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:52,682 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.051666666 minutes
|
||||
17-01-19 22:07:52,682 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:55,895 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.05165 minutes
|
||||
17-01-19 22:07:55,895 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:07:59,073 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.050816666 minutes
|
||||
17-01-19 22:07:59,073 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:02,072 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.047733333 minutes
|
||||
17-01-19 22:08:02,072 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:05,063 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0478 minutes
|
||||
17-01-19 22:08:05,063 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:08,138 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.048783332 minutes
|
||||
17-01-19 22:08:08,138 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:11,109 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.046933334 minutes
|
||||
17-01-19 22:08:11,109 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:13,900 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0434 minutes
|
||||
17-01-19 22:08:13,900 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:16,812 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.046066668 minutes
|
||||
17-01-19 22:08:16,813 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:19,637 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04458333 minutes
|
||||
17-01-19 22:08:19,637 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:22,261 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04135 minutes
|
||||
17-01-19 22:08:22,261 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:24,878 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041183334 minutes
|
||||
17-01-19 22:08:24,878 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:27,719 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.044633333 minutes
|
||||
17-01-19 22:08:27,720 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:30,388 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04191667 minutes
|
||||
17-01-19 22:08:30,388 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:33,013 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0411 minutes
|
||||
17-01-19 22:08:33,013 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:35,701 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.042066667 minutes
|
||||
17-01-19 22:08:35,701 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:38,561 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.044816665 minutes
|
||||
17-01-19 22:08:38,562 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:41,237 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041683335 minutes
|
||||
17-01-19 22:08:41,237 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:44,048 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.043916665 minutes
|
||||
17-01-19 22:08:44,048 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:46,638 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040333334 minutes
|
||||
17-01-19 22:08:46,638 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:49,601 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.046366666 minutes
|
||||
17-01-19 22:08:49,601 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:52,463 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04455 minutes
|
||||
17-01-19 22:08:52,464 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:55,288 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.043466665 minutes
|
||||
17-01-19 22:08:55,289 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:08:58,161 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.043983333 minutes
|
||||
17-01-19 22:08:58,161 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:00,883 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04215 minutes
|
||||
17-01-19 22:09:00,883 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:03,531 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0403 minutes
|
||||
17-01-19 22:09:03,531 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:06,165 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040683333 minutes
|
||||
17-01-19 22:09:06,165 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:08,824 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041016668 minutes
|
||||
17-01-19 22:09:08,824 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:11,631 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0428 minutes
|
||||
17-01-19 22:09:11,631 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:14,212 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039583333 minutes
|
||||
17-01-19 22:09:14,212 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:16,874 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04085 minutes
|
||||
17-01-19 22:09:16,874 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:19,474 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039883334 minutes
|
||||
17-01-19 22:09:19,474 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:22,269 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04305 minutes
|
||||
17-01-19 22:09:22,269 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:24,963 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04055 minutes
|
||||
17-01-19 22:09:24,963 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:27,618 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040583335 minutes
|
||||
17-01-19 22:09:27,618 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:30,175 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0385 minutes
|
||||
17-01-19 22:09:30,175 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:32,900 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041416667 minutes
|
||||
17-01-19 22:09:32,901 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:35,505 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039583333 minutes
|
||||
17-01-19 22:09:35,505 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:38,205 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040633332 minutes
|
||||
17-01-19 22:09:38,205 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:40,886 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0408 minutes
|
||||
17-01-19 22:09:40,886 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:43,660 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.042033333 minutes
|
||||
17-01-19 22:09:43,660 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:46,294 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03985 minutes
|
||||
17-01-19 22:09:46,294 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:49,021 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04146667 minutes
|
||||
17-01-19 22:09:49,021 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:51,649 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03975 minutes
|
||||
17-01-19 22:09:51,649 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:54,471 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04285 minutes
|
||||
17-01-19 22:09:54,471 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:57,111 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039666668 minutes
|
||||
17-01-19 22:09:57,111 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:09:59,755 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039683335 minutes
|
||||
17-01-19 22:09:59,755 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:02,441 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040216666 minutes
|
||||
17-01-19 22:10:02,441 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:05,232 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.042 minutes
|
||||
17-01-19 22:10:05,232 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:08,031 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04205 minutes
|
||||
17-01-19 22:10:08,032 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:11,085 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.045816667 minutes
|
||||
17-01-19 22:10:11,085 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:13,831 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041116666 minutes
|
||||
17-01-19 22:10:13,832 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:16,708 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04315 minutes
|
||||
17-01-19 22:10:16,708 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:19,451 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041116666 minutes
|
||||
17-01-19 22:10:19,451 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:22,132 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039883334 minutes
|
||||
17-01-19 22:10:22,132 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:24,828 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040133335 minutes
|
||||
17-01-19 22:10:24,828 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:27,833 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.044533335 minutes
|
||||
17-01-19 22:10:27,833 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:30,528 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039616667 minutes
|
||||
17-01-19 22:10:30,528 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:33,335 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041533332 minutes
|
||||
17-01-19 22:10:33,336 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:35,966 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03875 minutes
|
||||
17-01-19 22:10:35,966 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:38,964 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04485 minutes
|
||||
17-01-19 22:10:38,964 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:41,788 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04195 minutes
|
||||
17-01-19 22:10:41,788 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:44,709 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04256667 minutes
|
||||
17-01-19 22:10:44,709 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:47,312 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0382 minutes
|
||||
17-01-19 22:10:47,312 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:50,070 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040566668 minutes
|
||||
17-01-19 22:10:50,070 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:52,824 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0405 minutes
|
||||
17-01-19 22:10:52,824 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:55,547 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03973333 minutes
|
||||
17-01-19 22:10:55,547 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:10:58,283 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0402 minutes
|
||||
17-01-19 22:10:58,283 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:01,340 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04505 minutes
|
||||
17-01-19 22:11:01,340 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:04,041 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039483335 minutes
|
||||
17-01-19 22:11:04,041 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:06,711 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03875 minutes
|
||||
17-01-19 22:11:06,711 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:09,790 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.045533333 minutes
|
||||
17-01-19 22:11:09,790 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:12,664 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041666668 minutes
|
||||
17-01-19 22:11:12,665 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:15,393 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039616667 minutes
|
||||
17-01-19 22:11:15,393 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:18,046 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038283333 minutes
|
||||
17-01-19 22:11:18,047 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:20,994 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.043216668 minutes
|
||||
17-01-19 22:11:20,994 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:23,656 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03818333 minutes
|
||||
17-01-19 22:11:23,656 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:26,318 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038416665 minutes
|
||||
17-01-19 22:11:26,318 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:28,953 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0378 minutes
|
||||
17-01-19 22:11:28,953 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:32,079 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.045883335 minutes
|
||||
17-01-19 22:11:32,079 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:34,772 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03865 minutes
|
||||
17-01-19 22:11:34,773 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:37,559 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040033333 minutes
|
||||
17-01-19 22:11:37,559 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:40,233 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038116667 minutes
|
||||
17-01-19 22:11:40,233 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:43,268 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.044033334 minutes
|
||||
17-01-19 22:11:43,268 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:46,143 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04146667 minutes
|
||||
17-01-19 22:11:46,143 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:48,947 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04015 minutes
|
||||
17-01-19 22:11:48,947 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:51,839 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041666668 minutes
|
||||
17-01-19 22:11:51,839 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:54,944 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.045066666 minutes
|
||||
17-01-19 22:11:54,944 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:11:57,752 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0402 minutes
|
||||
17-01-19 22:11:57,752 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:00,688 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04191667 minutes
|
||||
17-01-19 22:12:00,688 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:03,568 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0412 minutes
|
||||
17-01-19 22:12:03,568 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:06,612 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.043966666 minutes
|
||||
17-01-19 22:12:06,612 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:09,319 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038283333 minutes
|
||||
17-01-19 22:12:09,319 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:12,326 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0431 minutes
|
||||
17-01-19 22:12:12,326 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:16,262 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.056283332 minutes
|
||||
17-01-19 22:12:16,263 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:19,965 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.054566666 minutes
|
||||
17-01-19 22:12:19,965 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:23,198 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.046816666 minutes
|
||||
17-01-19 22:12:23,198 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:26,243 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.043566667 minutes
|
||||
17-01-19 22:12:26,243 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:29,254 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.042316668 minutes
|
||||
17-01-19 22:12:29,254 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:32,280 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.042966668 minutes
|
||||
17-01-19 22:12:32,280 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:35,205 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04115 minutes
|
||||
17-01-19 22:12:35,205 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:37,973 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038483333 minutes
|
||||
17-01-19 22:12:37,973 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:40,739 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0388 minutes
|
||||
17-01-19 22:12:40,739 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:43,510 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0387 minutes
|
||||
17-01-19 22:12:43,511 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:46,204 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0372 minutes
|
||||
17-01-19 22:12:46,204 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:49,111 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04085 minutes
|
||||
17-01-19 22:12:49,111 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:51,832 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03775 minutes
|
||||
17-01-19 22:12:51,832 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:54,843 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0424 minutes
|
||||
17-01-19 22:12:54,843 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:12:58,052 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.045283332 minutes
|
||||
17-01-19 22:12:58,052 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:01,565 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.047616668 minutes
|
||||
17-01-19 22:13:01,565 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:04,603 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.042683333 minutes
|
||||
17-01-19 22:13:04,603 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:08,656 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.058066666 minutes
|
||||
17-01-19 22:13:08,657 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:13,159 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.061816666 minutes
|
||||
17-01-19 22:13:13,159 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:16,098 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04075 minutes
|
||||
17-01-19 22:13:16,098 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:19,037 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03965 minutes
|
||||
17-01-19 22:13:19,037 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:22,269 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.043916665 minutes
|
||||
17-01-19 22:13:22,269 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:25,147 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03985 minutes
|
||||
17-01-19 22:13:25,147 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:27,930 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038333334 minutes
|
||||
17-01-19 22:13:27,931 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:31,131 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.045266666 minutes
|
||||
17-01-19 22:13:31,131 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:34,170 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.042016666 minutes
|
||||
17-01-19 22:13:34,170 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:37,000 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038733333 minutes
|
||||
17-01-19 22:13:37,000 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:40,028 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.042083334 minutes
|
||||
17-01-19 22:13:40,028 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:42,996 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041116666 minutes
|
||||
17-01-19 22:13:42,996 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:45,830 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038766667 minutes
|
||||
17-01-19 22:13:45,830 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:48,743 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039816666 minutes
|
||||
17-01-19 22:13:48,743 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:51,834 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04305 minutes
|
||||
17-01-19 22:13:51,834 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:54,792 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040633332 minutes
|
||||
17-01-19 22:13:54,792 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:13:57,582 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038 minutes
|
||||
17-01-19 22:13:57,582 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:00,495 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039983332 minutes
|
||||
17-01-19 22:14:00,495 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:03,673 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.044016667 minutes
|
||||
17-01-19 22:14:03,673 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:06,957 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04605 minutes
|
||||
17-01-19 22:14:06,957 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:10,217 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04535 minutes
|
||||
17-01-19 22:14:10,217 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:13,819 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.051383335 minutes
|
||||
17-01-19 22:14:13,819 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:17,618 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.05455 minutes
|
||||
17-01-19 22:14:17,618 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:21,385 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.053816665 minutes
|
||||
17-01-19 22:14:21,385 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:25,536 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.059633333 minutes
|
||||
17-01-19 22:14:25,536 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:29,612 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0587 minutes
|
||||
17-01-19 22:14:29,612 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:33,353 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.0521 minutes
|
||||
17-01-19 22:14:33,353 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:36,280 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039516665 minutes
|
||||
17-01-19 22:14:36,280 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:39,265 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040433332 minutes
|
||||
17-01-19 22:14:39,265 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:42,743 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04898333 minutes
|
||||
17-01-19 22:14:42,743 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:45,794 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041333333 minutes
|
||||
17-01-19 22:14:45,795 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:48,922 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04285 minutes
|
||||
17-01-19 22:14:48,922 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:51,913 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040083334 minutes
|
||||
17-01-19 22:14:51,913 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:54,840 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.039433334 minutes
|
||||
17-01-19 22:14:54,840 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:14:57,899 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.041783333 minutes
|
||||
17-01-19 22:14:57,899 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:00,824 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03945 minutes
|
||||
17-01-19 22:15:00,824 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:03,802 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040333334 minutes
|
||||
17-01-19 22:15:03,802 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:06,777 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04025 minutes
|
||||
17-01-19 22:15:06,778 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:09,777 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.040733334 minutes
|
||||
17-01-19 22:15:09,777 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:12,615 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.037683334 minutes
|
||||
17-01-19 22:15:12,615 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:15,517 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038866665 minutes
|
||||
17-01-19 22:15:15,517 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:18,439 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.038933333 minutes
|
||||
17-01-19 22:15:18,439 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:21,539 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.04193333 minutes
|
||||
17-01-19 22:15:21,539 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:24,362 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.037266668 minutes
|
||||
17-01-19 22:15:24,362 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:27,206 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 1000 projects cost: 0.03745 minutes
|
||||
17-01-19 22:15:27,206 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:27,971 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:67) ## insert synonyms:deal with 95 projects cost: 0.0032 minutes
|
||||
17-01-19 22:15:27,971 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:46) ## ==============start insert synonyms============
|
||||
17-01-19 22:15:28,532 WARN com.ossean.InsertSynonyms(InsertSynonyms.java:50) ## insert synonyms:deal all projects FINALLY cost: 9.24365 minutes
|
||||
17-01-19 22:16:59,162 WARN com.ossean.GetSynonyms(GetSynonyms.java:51) ## ==============start insert synonyms============
|
||||
17-01-19 22:17:00,182 WARN com.ossean.GetSynonyms(GetSynonyms.java:55) ## insert synonyms:deal all projects FINALLY cost: 0.017 minutes
|
||||
17-01-19 22:22:00,183 WARN com.ossean.GetSynonyms(GetSynonyms.java:72) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 22:22:00,183 WARN com.ossean.GetSynonyms(GetSynonyms.java:51) ## ==============start insert synonyms============
|
||||
17-01-19 22:22:01,012 WARN com.ossean.GetSynonyms(GetSynonyms.java:55) ## insert synonyms:deal all projects FINALLY cost: 5.0308332 minutes
|
||||
17-01-19 22:27:01,013 WARN com.ossean.GetSynonyms(GetSynonyms.java:72) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 22:27:01,013 WARN com.ossean.GetSynonyms(GetSynonyms.java:51) ## ==============start insert synonyms============
|
||||
17-01-19 22:27:01,828 WARN com.ossean.GetSynonyms(GetSynonyms.java:55) ## insert synonyms:deal all projects FINALLY cost: 10.044434 minutes
|
||||
17-01-19 22:32:01,828 WARN com.ossean.GetSynonyms(GetSynonyms.java:72) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 22:32:01,828 WARN com.ossean.GetSynonyms(GetSynonyms.java:51) ## ==============start insert synonyms============
|
||||
17-01-19 22:32:02,613 WARN com.ossean.GetSynonyms(GetSynonyms.java:55) ## insert synonyms:deal all projects FINALLY cost: 15.057517 minutes
|
||||
17-01-19 22:34:35,817 WARN com.ossean.MergeProjects(MergeProjects.java:78) ## deal with:171000 projects cost: 17.6093 minutes
|
||||
17-01-19 22:35:50,586 WARN com.ossean.MergeProjects(MergeProjects.java:62) ## deal with 183095 projects cost: 18.85545 minutes
|
||||
17-01-19 22:37:02,615 WARN com.ossean.GetSynonyms(GetSynonyms.java:72) ## insert synonyms:deal with 0 projects cost: 0.0 minutes
|
||||
17-01-19 22:37:02,615 WARN com.ossean.GetSynonyms(GetSynonyms.java:51) ## ==============start insert synonyms============
|
||||
17-01-19 22:37:03,208 WARN com.ossean.GetSynonyms(GetSynonyms.java:55) ## insert synonyms:deal all projects FINALLY cost: 20.067432 minutes
|
||||
|
|
@ -47,7 +47,7 @@
|
|||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.10</version>
|
||||
<scope>compile</scope>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
@ -56,12 +56,6 @@
|
|||
<version>3.1</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>info.debatty</groupId>
|
||||
<artifactId>java-string-similarity</artifactId>
|
||||
<version>RELEASE</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-collections</groupId>
|
||||
<artifactId>commons-collections</artifactId>
|
||||
|
|
@ -130,7 +124,7 @@
|
|||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-test</artifactId>
|
||||
<version>${spring-version}</version>
|
||||
<scope>compile </scope>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,100 @@
|
|||
package com.ossean;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.databaseSource.DBSource;
|
||||
import com.ossean.model.OpenSourceProject;
|
||||
|
||||
|
||||
@Component
|
||||
public class ChangeSynonyms {
|
||||
|
||||
Logger logger = Logger.getLogger(this.getClass());
|
||||
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
|
||||
private static String sourceTableName = "open_source_projects";
|
||||
private static String targetTableName = "synonyms";
|
||||
private static String pointerTableName = "edd_pointers";
|
||||
|
||||
private static int batchSize = 1000;
|
||||
|
||||
|
||||
//读指针
|
||||
public int readPointer(String table, String source, String target){
|
||||
int pointer = 1;
|
||||
//pointer的初始值应该从数据库中读出最小的id
|
||||
try {
|
||||
pointer = dbSource.getMinId(source);
|
||||
} catch (Exception e){
|
||||
//表示表中没有数据
|
||||
logger.info("No item in this table");
|
||||
dbSource.insertPointer(table, source, target, 1);
|
||||
return 1;
|
||||
}
|
||||
|
||||
try {
|
||||
pointer = dbSource.getPointer(table, source, target);
|
||||
} catch(Exception e) {
|
||||
logger.info("No such pointer! Create one");
|
||||
dbSource.insertPointer(table, source, target, pointer);
|
||||
}
|
||||
return pointer;
|
||||
}
|
||||
|
||||
|
||||
public void start(){
|
||||
while(true){
|
||||
//读取对应的指针
|
||||
int startId = readPointer(pointerTableName, sourceTableName, targetTableName);
|
||||
List<OpenSourceProject> openSourceProjectList = dbSource.getOpenSourceProjectList(sourceTableName, startId, batchSize);
|
||||
|
||||
if(openSourceProjectList.size() == 0){
|
||||
//表示程序执行完成
|
||||
logger.info("程序执行完成");
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
for(OpenSourceProject project:openSourceProjectList){
|
||||
String synonyms = project.getSynonyms();
|
||||
logger.info("处理项目:" + project.getId());
|
||||
//判断synonyms是否需要修改
|
||||
if(synonyms == null || "".equals(synonyms)){
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, project.getId() + 1); //更新指针
|
||||
continue; //表示没有同义词 不需要修改
|
||||
}
|
||||
|
||||
if(synonyms.indexOf("<") >= 0){
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, project.getId() + 1); //更新指针
|
||||
continue; //表示存储的是正确的结果 不需要修改
|
||||
}else{
|
||||
logger.info("修改项目:" + project.getId());
|
||||
String result = "";
|
||||
String[] array = synonyms.split(",");
|
||||
for(String s:array){
|
||||
result = result + "<" + s + ">,";
|
||||
}
|
||||
result = result.substring(0, result.length() - 1);
|
||||
dbSource.updateSynonyms(sourceTableName, result, project.getId());
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, project.getId() + 1); //更新指针
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args){
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
ChangeSynonyms Main = applicationContext.getBean(ChangeSynonyms.class);
|
||||
Main.start();
|
||||
}
|
||||
}
|
||||
|
|
@ -13,19 +13,21 @@ import org.springframework.stereotype.Component;
|
|||
|
||||
@Component
|
||||
public class ClearTable {
|
||||
|
||||
Logger logger = Logger.getLogger(ClearTable.class);
|
||||
public void start(){
|
||||
truncateTable(TableName.eddRelationTableName);
|
||||
String sql = "update synonymmings_2017 set flag=null;";
|
||||
update(sql);
|
||||
String sql2 = "update gather_projects set update_mark=0;";
|
||||
update(sql2);
|
||||
deleteItemInEddPointers(TableName.gatherProjectsTableName, TableName.eddRelationTableName);
|
||||
logger.info("完成去重程序运行前数据表的清空");
|
||||
truncateTable("synonyms");
|
||||
truncateTable("edd_relations");
|
||||
truncateTable("synonymmings");
|
||||
truncateTable("open_source_projects");
|
||||
deleteItemInEddPointers("gather_projects", "edd_relations");
|
||||
//deleteTaggings4Project();
|
||||
|
||||
logger.info("完成去重程序运行前数据表的清空和taggings表OpenSourceProject的删除");
|
||||
}
|
||||
//删除edd_pointers表中transfer对应的数据
|
||||
public void deleteItemInEddPointers(String sourceTableName, String targetTableName){
|
||||
String sql = "delete from edd_pointers_17 where SourceTableName=? and TargetTableName=?";
|
||||
String sql = "delete from edd_pointers where SourceTableName=? and TargetTableName=?";
|
||||
Connection conn = getConnection();
|
||||
if(conn == null){
|
||||
logger.info("没有获取到Connection");
|
||||
|
|
@ -61,41 +63,6 @@ public class ClearTable {
|
|||
}
|
||||
}
|
||||
|
||||
public void updateMark(String table){
|
||||
String sql = "update " + table+" set update_mark=1;";
|
||||
Connection conn = getConnection();
|
||||
if(conn == null){
|
||||
logger.info("没有获取到Connection");
|
||||
System.exit(-1);
|
||||
}
|
||||
PreparedStatement ps;
|
||||
try {
|
||||
ps = conn.prepareStatement(sql);
|
||||
ps.execute();
|
||||
ps.close();
|
||||
conn.close();
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public void update(String sql){
|
||||
Connection conn = getConnection();
|
||||
if(conn == null){
|
||||
logger.info("没有获取到Connection");
|
||||
System.exit(-1);
|
||||
}
|
||||
PreparedStatement ps;
|
||||
try {
|
||||
ps = conn.prepareStatement(sql);
|
||||
ps.execute();
|
||||
ps.close();
|
||||
conn.close();
|
||||
} catch (SQLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteTaggings4Project(){
|
||||
String sql = "delete from taggings where taggable_type=?";
|
||||
Connection conn = getConnection();
|
||||
|
|
@ -118,7 +85,7 @@ public class ClearTable {
|
|||
Connection conn = null;
|
||||
try {
|
||||
Class.forName("com.mysql.jdbc.Driver").newInstance();
|
||||
conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/ossean", "root", "123456");
|
||||
conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/ossean_production", "root", "NUDTpdl@123");
|
||||
} catch (InstantiationException e) {
|
||||
e.printStackTrace();
|
||||
} catch (IllegalAccessException e) {
|
||||
|
|
|
|||
|
|
@ -16,9 +16,10 @@ public class ClearTableBeforTransfer {
|
|||
|
||||
Logger logger = Logger.getLogger(ClearTableBeforTransfer.class);
|
||||
public void start(){
|
||||
truncateTable(TableName.openSourceProjectsTableName);
|
||||
deleteItemInEddPointers(TableName.eddRelationTableName, TableName.openSourceProjectsTableName);
|
||||
deleteTaggingsForProject(TableName.taggingsTableName);
|
||||
truncateTable("open_source_projects");
|
||||
deleteItemInEddPointers("edd_relations", "open_source_projects");
|
||||
deleteTaggings4Project();
|
||||
|
||||
logger.info("完成转移程序运行前数据表的清空和taggings表OpenSourceProject的删除");
|
||||
}
|
||||
|
||||
|
|
@ -36,12 +37,13 @@ public class ClearTableBeforTransfer {
|
|||
ps.close();
|
||||
conn.close();
|
||||
} catch (SQLException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public void deleteTaggingsForProject(String taggingsTableName){
|
||||
String sql = "delete from "+taggingsTableName+" where taggable_type=?";
|
||||
public void deleteTaggings4Project(){
|
||||
String sql = "delete from taggings where taggable_type=?";
|
||||
Connection conn = getConnection();
|
||||
if(conn == null){
|
||||
logger.info("没有获取到Connection");
|
||||
|
|
@ -54,9 +56,11 @@ public class ClearTableBeforTransfer {
|
|||
ps.close();
|
||||
conn.close();
|
||||
} catch (SQLException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
//删除edd_pointers表中transfer对应的数据
|
||||
public void deleteItemInEddPointers(String sourceTableName, String targetTableName){
|
||||
String sql = "delete from edd_pointers where SourceTableName=? and TargetTableName=?";
|
||||
|
|
@ -73,6 +77,7 @@ public class ClearTableBeforTransfer {
|
|||
ps.close();
|
||||
conn.close();
|
||||
} catch (SQLException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
|
@ -81,14 +86,20 @@ public class ClearTableBeforTransfer {
|
|||
Connection conn = null;
|
||||
try {
|
||||
Class.forName("com.mysql.jdbc.Driver").newInstance();
|
||||
conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/ossean", "root", "123456");
|
||||
conn = DriverManager.getConnection("jdbc:mysql://172.16.128.30:3306/ossean_production", "gather", "influx1234");
|
||||
|
||||
// conn = DriverManager.getConnection("jdbc:mysql://localhost:3306/extract_result", "root", "123456");
|
||||
} catch (InstantiationException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (IllegalAccessException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (ClassNotFoundException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (SQLException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
return conn;
|
||||
|
|
|
|||
|
|
@ -1,339 +0,0 @@
|
|||
package com.ossean;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.dao.DBSource;
|
||||
import com.ossean.dao.GatherDao;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.util.EDDHandler;
|
||||
import com.ossean.util.FileReader;
|
||||
import com.ossean.util.RegexHandler;
|
||||
import com.ossean.util.StringHandler;
|
||||
import com.ossean.util.StringHandler_ProjectName;
|
||||
|
||||
@Component("getSynonyms")
|
||||
public class GetSynonyms extends Thread{
|
||||
|
||||
Logger logger = Logger.getLogger(this.getClass());
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
|
||||
private static String gatherProjectsTableName = TableName.gatherProjectsTableName;
|
||||
private static String synonymsTableName = TableName.synonymsTableName;
|
||||
private static String pointerTableName = TableName.pointerTableName;
|
||||
private static int batchSize = 1000;
|
||||
|
||||
@Override
|
||||
public void run(){
|
||||
long start_all = System.currentTimeMillis();
|
||||
|
||||
while(true){
|
||||
/*
|
||||
* update_mark:0 原始Gather_projects数据
|
||||
* update_mark：1取完别名
|
||||
* update_mark：2去完重
|
||||
* 若是更新完的数据则更新update_mark字段为0
|
||||
*/
|
||||
logger.warn("==============start insert synonyms============");
|
||||
List<GatherProjectsModel> gpmList = gatherDao.selectGPMListNew(gatherProjectsTableName,0,batchSize);
|
||||
if(gpmList.size() == 0){
|
||||
try {
|
||||
logger.warn("insert synonyms:deal all projects FINALLY cost: "+(float)(System.currentTimeMillis() - start_all)/60000+" minutes");
|
||||
logger.info("all gather project have get synonyms. Sleep 5 mins");
|
||||
Thread.sleep(1000*300);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
long start = System.currentTimeMillis();
|
||||
for(GatherProjectsModel model:gpmList){
|
||||
Map<String,Integer> synonymsList = getSynonyms(model);
|
||||
for(String name:synonymsList.keySet()){
|
||||
dbSource.insertSynonymsNew(synonymsTableName,model.getId(),name,synonymsList.get(name));
|
||||
}
|
||||
//更新update_mark=1
|
||||
gatherDao.updateMark(gatherProjectsTableName,1, model.getId());
|
||||
}
|
||||
//long end = System.currentTimeMillis();
|
||||
//logger.warn("insert synonyms:deal with "+gpmList.size()+" projects cost: "+(float)(end - start)/60000+" minutes");
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
InsertSynonyms Main = applicationContext.getBean(InsertSynonyms.class);
|
||||
Main.start();
|
||||
|
||||
}
|
||||
public Map<String,Integer> getSynonyms(GatherProjectsModel model){
|
||||
|
||||
model.setName(model.getName().trim());
|
||||
model.setDescription(model.getDescription().trim());
|
||||
Map<String,Integer> SynonymMap = new HashMap<String,Integer>();//记录每个别名及其准确度，<Sysnonym,1>表时Synonym可靠性很大
|
||||
List<String> synonymsList = new ArrayList<String>();//最终返回的同义词列表
|
||||
List<String> synonymsList_notsure = new ArrayList<String>();
|
||||
String description = model.getDescription();
|
||||
String prjName = model.getName();
|
||||
|
||||
List<String> result = null;
|
||||
if(description == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
List<String> linkNameList = new ArrayList<String>();//用于存储所有提取出来的关联项目名
|
||||
List<String> keywords = FileReader.read("./files/language.txt");
|
||||
|
||||
//对项目名进行处理,去除公司名或基金组织的信息&&去除括号（没有公司没有括号则包含自己）
|
||||
String projectNameWithoutBracket = StringHandler_ProjectName.removeBracket(prjName);
|
||||
String projectNameWithoutComName = StringHandler_ProjectName.getProjectWithoutComName(projectNameWithoutBracket);
|
||||
//String projectNameWithoutComName = StringHandler_ProjectName.getProjectWithoutComName(prjName);
|
||||
//projectNameWithoutComName = StringHandler_ProjectName.removeBracket(projectNameWithoutComName);
|
||||
if(!"".equals(projectNameWithoutComName)){
|
||||
synonymsList.add(projectNameWithoutComName);
|
||||
//logger.info("gets synonyms:" + projectNameWithoutComName + " Without ComName and remove racket");
|
||||
}
|
||||
|
||||
//提取项目名括号中的信息，不是项目类型
|
||||
String bracket = StringHandler_ProjectName.getBracket(prjName);
|
||||
if(bracket != null &&!"".equals(bracket)){
|
||||
bracket = RegexHandler.extractEngDecChiAndDot(bracket);//去除括号中信息的特殊符号
|
||||
if(!RegexHandler.onlySpecialChar(bracket)){
|
||||
if(!keywords.contains(bracket.toLowerCase())){//括号中的信息不是语言类关键字词
|
||||
if(StringHandler.isFullOrShort(bracket, projectNameWithoutBracket))
|
||||
{
|
||||
synonymsList.add(bracket);
|
||||
}
|
||||
else
|
||||
{
|
||||
if(StringHandler.specialIsFullOrShort(bracket, projectNameWithoutBracket))
|
||||
synonymsList.add(bracket);
|
||||
else linkNameList.add(bracket);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(description != null){
|
||||
|
||||
List<String> beWords = FileReader.read("./files/beVerb.txt");//读取be动词表
|
||||
description = StringHandler.getFirstSentence(description);
|
||||
|
||||
for(String beWord:beWords){
|
||||
|
||||
String linkName = StringHandler.findLinkName(description, beWord);//按系动词表优先级进行匹配
|
||||
if(linkName == null){//表示没有匹配到当前的系动词 进行下一个匹配
|
||||
continue;
|
||||
}
|
||||
if(linkName.toLowerCase().equals(prjName.toLowerCase()))
|
||||
continue;
|
||||
//特征短语去噪
|
||||
List<String> linkNameResult = EDDHandler.getMainAndViceDescriptionByBracket(linkName);//提取主副描述信息(括号中)
|
||||
linkNameResult = EDDHandler.getMainAndViceDescriptionByKnowAs(linkNameResult);//提取主副描述信息(known as+连词)
|
||||
linkNameResult = EDDHandler.removeComma(linkNameResult);//去除定语描述信息(逗号定语)
|
||||
linkNameResult = EDDHandler.removeDemonstrativeWords(linkNameResult);//去除定语描述信息(定冠词The)
|
||||
linkNameResult = EDDHandler.removeArrtibutiveClause(linkNameResult);//去除从句描述信息
|
||||
linkNameResult = EDDHandler.removePronoun(linkNameResult);//去除指示代词
|
||||
linkNameResult = EDDHandler.removeProject(linkNameResult);//去除项目常用词
|
||||
linkNameResult = EDDHandler.removePPCombine(linkNameResult);//去除指示代词和项目常用词两者的笛卡尔积
|
||||
linkNameResult = RegexHandler.extractEngDecChiAndDot(linkNameResult);//提取项目中
|
||||
linkNameResult = RegexHandler.removeOnlySpecial(linkNameResult);//删除只有特殊字符的同义词
|
||||
for(String name:linkNameResult){
|
||||
//logger.info("be 动词之前的有： "+name);
|
||||
//be动词前不包括项目类型，且和项目名不相同
|
||||
if(!keywords.contains(name.toLowerCase().trim()) && !name.toLowerCase().trim().equals(prjName.toLowerCase().trim())
|
||||
&& !name.toLowerCase().trim().equals(projectNameWithoutComName.toLowerCase().trim())){
|
||||
//这里的prjName是完整的name
|
||||
if(prjName.toLowerCase().indexOf(name.toLowerCase()) != -1){
|
||||
String withoutShortDash = RemoveShortDash(name);
|
||||
if(name.toLowerCase().equals(bracket.toLowerCase()) || name.toLowerCase().equals(withoutShortDash))
|
||||
synonymsList.add(name);
|
||||
else
|
||||
{
|
||||
int nameLength1 = withoutShortDash.split(" ").length;
|
||||
int nameLength2 = projectNameWithoutBracket.split(" ").length;
|
||||
int nameLength = name.trim().split(" ").length;
|
||||
double ratio1= (nameLength1/nameLength) * 1.0;
|
||||
double ratio2 =(nameLength2/nameLength) * 1.0;
|
||||
if(nameLength1 > 1 && ratio1 > 0.6)
|
||||
synonymsList.add(name);
|
||||
else
|
||||
{
|
||||
if(nameLength2 > 1 && ratio2 > 0.6)
|
||||
synonymsList.add(name);
|
||||
else linkNameList.add(name);
|
||||
}
|
||||
}
|
||||
|
||||
//synonymsList.add(name);//问题出在这个地方，如果从discription中提取出的词在名字中有出现就直接加入，这
|
||||
//会导致比如Ajax被识别为Ajax Implement的别名，而实际上并不是这样的。
|
||||
//logger.info("gets synonyms:" + name + " from description before be verb");
|
||||
}
|
||||
else
|
||||
linkNameList.add(name);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
//result包含去掉公司信息、括号内的、be动词之前的
|
||||
result = removeDupStr(linkNameList);
|
||||
//logger.info("result去掉公司信息、括号内的、be动词之前的;");
|
||||
for (int i = 0;i<result.size();i++)
|
||||
System.out.println("result: "+result.get(i));
|
||||
|
||||
//去除重复的
|
||||
|
||||
|
||||
String fullName = StringHandler.getFullName(projectNameWithoutComName.toLowerCase(), result);
|
||||
if(!fullName.equals("")){
|
||||
synonymsList.add(fullName.toLowerCase());//表示项目名确实是缩写 并提取到了项目全名
|
||||
//logger.info("项目的全称为： "+fullName);
|
||||
}
|
||||
|
||||
|
||||
String shortName = StringHandler.getShortName(projectNameWithoutComName.toLowerCase(), result);
|
||||
if(!shortName.equals("")){
|
||||
synonymsList.add(shortName.toLowerCase());//表示项目名确实是全称 并提取到了项目缩写
|
||||
//logger.info("项目的简称为： "+shortName);
|
||||
}
|
||||
|
||||
//同义词要和项目名称有一定的相似度
|
||||
for(int i = 0; i < result.size(); i++){
|
||||
String extract = "";
|
||||
String synonym = result.get(i);
|
||||
String[] words = synonym.split(" ");//按照空格进行分词
|
||||
for(String word:words){
|
||||
if(model.getName().toLowerCase().indexOf(word.toLowerCase()) >= 0){//查看每个词在项目名中是否出现
|
||||
extract += word + " ";
|
||||
}
|
||||
}
|
||||
if(extract.length() !=0)
|
||||
extract = extract.substring(0, extract.length() - 1);
|
||||
//logger.info("extract : "+extract);
|
||||
if(!"".equals(extract) && !keywords.contains(extract)){
|
||||
//提取的别名不和项目名重复
|
||||
if(!extract.toLowerCase().equals(prjName.toLowerCase().trim()) && !extract.toLowerCase().trim().equals(projectNameWithoutComName.toLowerCase().trim())){
|
||||
double similarDegree = SimilarDegree(extract,projectNameWithoutComName);
|
||||
System.out.println("字符串间的相似度为: "+similarDegree);
|
||||
if(similarDegree >= 0.4 && similarDegree <1.0){
|
||||
//logger.info("from extract after similar： "+extract);
|
||||
synonymsList_notsure.add(extract);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
removeDupStr(synonymsList);//去除同名列表中重复的
|
||||
removeDupStr(synonymsList_notsure);
|
||||
for(String s : synonymsList){
|
||||
//logger.info("finally synonyms:" + s + " by description");
|
||||
}
|
||||
|
||||
for(String sure:synonymsList)
|
||||
SynonymMap.put(sure, 1);
|
||||
for(String Notsure:synonymsList_notsure)
|
||||
SynonymMap.put(Notsure, 0);
|
||||
return SynonymMap;
|
||||
}
|
||||
|
||||
|
||||
public static String RemoveShortDash(String name)
|
||||
{
|
||||
String withoutShortDash="";
|
||||
if(name.indexOf('-') > -1)
|
||||
withoutShortDash = name.substring(0,name.indexOf('-')).trim();
|
||||
return withoutShortDash;
|
||||
}
|
||||
public List<String> removeDupStr(List<String> array){
|
||||
List<String> list = new ArrayList<String>();
|
||||
Set<String> set = new HashSet<String>();
|
||||
for(String s : array){
|
||||
if(set.add(s.toLowerCase())){
|
||||
list.add(s);
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
public static double SimilarDegree(String strA, String strB){
|
||||
|
||||
String newStrA = removeSign(strA);
|
||||
String newStrB = removeSign(strB);
|
||||
int temp = Math.max(newStrA.length(), newStrB.length());
|
||||
int temp2 = longestCommonSubstring(newStrA, newStrB).length();
|
||||
return temp2 * 1.0 / temp;
|
||||
|
||||
}
|
||||
private static String removeSign(String str) {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
for (char item : str.toCharArray())
|
||||
if (charReg(item)){
|
||||
sb.append(item);
|
||||
}
|
||||
return sb.toString();
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static boolean charReg(char charValue) {
|
||||
|
||||
return (charValue >= 0x4E00 && charValue <= 0X9FA5)
|
||||
|| (charValue >= 'a' && charValue <= 'z')
|
||||
|| (charValue >= 'A' && charValue <= 'Z')
|
||||
|| (charValue >= '0' && charValue <= '9');
|
||||
|
||||
}
|
||||
|
||||
private static String longestCommonSubstring(String strA, String strB) {
|
||||
char[] chars_strA = strA.toCharArray();
|
||||
char[] chars_strB = strB.toCharArray();
|
||||
int m = chars_strA.length;
|
||||
int n = chars_strB.length;
|
||||
int[][] matrix = new int[m + 1][n + 1];
|
||||
for (int i = 1; i <= m; i++) {
|
||||
|
||||
for (int j = 1; j <= n; j++) {
|
||||
if (chars_strA[i - 1] == chars_strB[j - 1])
|
||||
matrix[i][j] = matrix[i - 1][j - 1] + 1;
|
||||
else
|
||||
matrix[i][j] = Math.max(matrix[i][j - 1], matrix[i - 1][j]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
char[] result = new char[matrix[m][n]];
|
||||
int currentIndex = result.length - 1;
|
||||
while (matrix[m][n] != 0) {
|
||||
if (matrix[m][n] == matrix[m][n - 1])
|
||||
n--;
|
||||
else if (matrix[m][n] == matrix[m - 1][n])
|
||||
m--;
|
||||
else {
|
||||
result[currentIndex] = chars_strA[m - 1];
|
||||
currentIndex--;
|
||||
n--;
|
||||
m--;
|
||||
}
|
||||
}
|
||||
return new String(result);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -1,282 +0,0 @@
|
|||
package com.ossean;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import javax.annotation.Resource;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
import com.ossean.dao.DBSource;
|
||||
import com.ossean.dao.GatherDao;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.util.EDDHandler;
|
||||
import com.ossean.util.FileReader;
|
||||
import com.ossean.util.RegexHandler;
|
||||
import com.ossean.util.StringHandler;
|
||||
import com.ossean.util.StringHandler_ProjectName;
|
||||
|
||||
@Component("insertSynonyms")
|
||||
public class InsertSynonyms extends Thread{
|
||||
|
||||
Logger logger = Logger.getLogger(this.getClass());
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
|
||||
private static String gatherProjectsTableName = TableName.gatherProjectsTableName;
|
||||
private static String synonymsTableName = TableName.synonymsTableName;
|
||||
private static String pointerTableName = TableName.pointerTableName;
|
||||
private static int batchSize = 1000;
|
||||
|
||||
@Override
|
||||
public void run(){
|
||||
long start_all = System.currentTimeMillis();
|
||||
|
||||
while(true){
|
||||
/*
|
||||
* update_mark:0 原始Gather_projects数据
|
||||
* update_mark：1取完别名
|
||||
* update_mark：2去完重
|
||||
* 若是更新完的数据则更新update_mark字段为0
|
||||
*/
|
||||
logger.warn("==============start insert synonyms============");
|
||||
List<GatherProjectsModel> gpmList = gatherDao.selectGPMListNew(gatherProjectsTableName,0,batchSize);
|
||||
if(gpmList.size() == 0){
|
||||
try {
|
||||
logger.warn("insert synonyms:deal all projects FINALLY cost: "+(float)(System.currentTimeMillis() - start_all)/60000+" minutes");
|
||||
logger.info("all gather project have get synonyms. Sleep 5 mins");
|
||||
Thread.sleep(1000*300);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
long start = System.currentTimeMillis();
|
||||
for(GatherProjectsModel model:gpmList){
|
||||
List<String> synonymsList = getSynonyms(model);
|
||||
for(String name:synonymsList){
|
||||
dbSource.insertSynonymsNew(synonymsTableName,model.getId(),name,1);
|
||||
}
|
||||
//更新update_mark=1
|
||||
gatherDao.updateMark(gatherProjectsTableName,1, model.getId());
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
logger.warn("insert synonyms:deal with "+gpmList.size()+" projects cost: "+(float)(end - start)/60000+" minutes");
|
||||
}
|
||||
}
|
||||
|
||||
// public static void main(String[] args){
|
||||
// ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
// InsertSynonyms Main = applicationContext.getBean(InsertSynonyms.class);
|
||||
//
|
||||
// Main.start();
|
||||
//
|
||||
// }
|
||||
|
||||
public List<String> getSynonyms(GatherProjectsModel model){
|
||||
|
||||
model.setName(model.getName().trim());
|
||||
model.setDescription(model.getDescription().trim());
|
||||
|
||||
List<String> synonymsList = new ArrayList<String>();//最终返回的同义词列表
|
||||
String description = model.getDescription();
|
||||
String prjName = model.getName();
|
||||
List<String> result = null;
|
||||
|
||||
if(description == null)
|
||||
return result;
|
||||
List<String> linkNameList = new ArrayList<String>();//用于存储所有提取出来的关联项目名
|
||||
List<String> keywords = FileReader.read("./files/language.txt");
|
||||
|
||||
//提取项目名括号中的信息，不是项目类型
|
||||
String bracket = StringHandler_ProjectName.getBracket(prjName);
|
||||
if(bracket != null &&!"".equals(bracket)){
|
||||
bracket = RegexHandler.extractEngDecChiAndDot(bracket);//去除括号中信息的特殊符号
|
||||
if(!RegexHandler.onlySpecialChar(bracket)){
|
||||
if(!keywords.contains(bracket.toLowerCase())){//括号中的信息不是语言类关键字词
|
||||
linkNameList.add(bracket);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//对项目名进行处理,去除公司名或基金组织的信息&&去除括号（没有公司没有括号则包含自己）
|
||||
String projectNameWithoutComName = StringHandler_ProjectName.getProjectWithoutComName(prjName);
|
||||
projectNameWithoutComName = StringHandler_ProjectName.removeBracket(projectNameWithoutComName);
|
||||
if(!"".equals(projectNameWithoutComName)){
|
||||
synonymsList.add(projectNameWithoutComName);
|
||||
logger.info("gets synonyms:" + projectNameWithoutComName + " Without ComName and remove racket");
|
||||
}
|
||||
|
||||
|
||||
if(description != null){
|
||||
|
||||
List<String> beWords = FileReader.read("./files/beVerb.txt");//读取be动词表
|
||||
description = StringHandler.getFirstSentence(description);
|
||||
|
||||
for(String beWord:beWords){
|
||||
|
||||
String linkName = StringHandler.findLinkName(description, beWord);//按系动词表优先级进行匹配
|
||||
if(linkName == null){//表示没有匹配到当前的系动词 进行下一个匹配
|
||||
continue;
|
||||
}
|
||||
//特征短语去噪
|
||||
List<String> linkNameResult = EDDHandler.getMainAndViceDescriptionByBracket(linkName);//提取主副描述信息(括号中)
|
||||
linkNameResult = EDDHandler.getMainAndViceDescriptionByKnowAs(linkNameResult);//提取主副描述信息(known as+连词)
|
||||
linkNameResult = EDDHandler.removeComma(linkNameResult);//去除定语描述信息(逗号定语)
|
||||
linkNameResult = EDDHandler.removeDemonstrativeWords(linkNameResult);//去除定语描述信息(定冠词The)
|
||||
linkNameResult = EDDHandler.removeArrtibutiveClause(linkNameResult);//去除从句描述信息
|
||||
linkNameResult = EDDHandler.removePronoun(linkNameResult);//去除指示代词
|
||||
linkNameResult = EDDHandler.removeProject(linkNameResult);//去除项目常用词
|
||||
linkNameResult = EDDHandler.removePPCombine(linkNameResult);//去除指示代词和项目常用词两者的笛卡尔积
|
||||
linkNameResult = RegexHandler.extractEngDecChiAndDot(linkNameResult);//提取项目中
|
||||
linkNameResult = RegexHandler.removeOnlySpecial(linkNameResult);//删除只有特殊字符的同义词
|
||||
for(String name:linkNameResult){
|
||||
logger.info("be 动词之前的有： "+name);
|
||||
//be动词前不包括项目类型，且和项目名不相同
|
||||
if(!keywords.contains(name.toLowerCase().trim()) && !name.toLowerCase().trim().equals(prjName.toLowerCase().trim())
|
||||
&& !name.toLowerCase().trim().equals(projectNameWithoutComName.toLowerCase().trim())){
|
||||
if(prjName.toLowerCase().indexOf(name.toLowerCase()) != -1){
|
||||
synonymsList.add(name);
|
||||
logger.info("gets synonyms:" + name + " from description before be verb");
|
||||
}
|
||||
else
|
||||
linkNameList.add(name);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//result包含去掉公司信息、括号内的、be动词之前的
|
||||
result = removeDupStr(linkNameList);
|
||||
logger.info("result去掉公司信息、括号内的、be动词之前的;");
|
||||
for (int i = 0;i<result.size();i++)
|
||||
System.out.println("result: "+result.get(i));
|
||||
|
||||
String fullName = StringHandler.getFullName(projectNameWithoutComName.toLowerCase(), result);
|
||||
if(!fullName.equals("")){
|
||||
synonymsList.add(fullName.toLowerCase());//表示项目名确实是缩写 并提取到了项目全名
|
||||
logger.info("项目的全称为： "+fullName);
|
||||
}
|
||||
|
||||
|
||||
String shortName = StringHandler.getShortName(projectNameWithoutComName.toLowerCase(), result);
|
||||
if(!shortName.equals("")){
|
||||
synonymsList.add(shortName.toLowerCase());//表示项目名确实是全称 并提取到了项目缩写
|
||||
logger.info("项目的简称为： "+shortName);
|
||||
}
|
||||
|
||||
//同义词要和项目名称有一定的相似度
|
||||
for(int i = 0; i < result.size(); i++){
|
||||
String extract = "";
|
||||
String synonym = result.get(i);
|
||||
String[] words = synonym.split(" ");//按照空格进行分词
|
||||
for(String word:words){
|
||||
if(model.getName().toLowerCase().indexOf(word.toLowerCase()) >= 0){//查看每个词在项目名中是否出现
|
||||
extract += word + " ";
|
||||
}
|
||||
}
|
||||
if(extract.length() !=0)
|
||||
extract = extract.substring(0, extract.length() - 1);
|
||||
//logger.info("extract : "+extract);
|
||||
if(!"".equals(extract) && !keywords.contains(extract)){
|
||||
//提取的别名不和项目名重复
|
||||
if(!extract.toLowerCase().equals(prjName.toLowerCase().trim()) && !extract.toLowerCase().trim().equals(projectNameWithoutComName.toLowerCase().trim())){
|
||||
double similarDegree = SimilarDegree(extract,projectNameWithoutComName);
|
||||
System.out.println("字符串间的相似度为: "+similarDegree);
|
||||
if(similarDegree >= 0.4 && similarDegree <1.0){
|
||||
logger.info("from extract after similar： "+extract);
|
||||
synonymsList.add(extract);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
removeDupStr(synonymsList);//去除同名列表中重复的
|
||||
for(String s : synonymsList){
|
||||
logger.info("finally synonyms:" + s + " by description");
|
||||
}
|
||||
return synonymsList;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public List<String> removeDupStr(List<String> array){
|
||||
List<String> list = new ArrayList<String>();
|
||||
Set<String> set = new HashSet<String>();
|
||||
for(String s : array){
|
||||
if(set.add(s.toLowerCase())){
|
||||
list.add(s);
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
public static double SimilarDegree(String strA, String strB){
|
||||
|
||||
String newStrA = removeSign(strA);
|
||||
String newStrB = removeSign(strB);
|
||||
int temp = Math.max(newStrA.length(), newStrB.length());
|
||||
int temp2 = longestCommonSubstring(newStrA, newStrB).length();
|
||||
return temp2 * 1.0 / temp;
|
||||
|
||||
}
|
||||
private static String removeSign(String str) {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
for (char item : str.toCharArray())
|
||||
if (charReg(item)){
|
||||
sb.append(item);
|
||||
}
|
||||
return sb.toString();
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static boolean charReg(char charValue) {
|
||||
|
||||
return (charValue >= 0x4E00 && charValue <= 0X9FA5)
|
||||
|| (charValue >= 'a' && charValue <= 'z')
|
||||
|| (charValue >= 'A' && charValue <= 'Z')
|
||||
|| (charValue >= '0' && charValue <= '9');
|
||||
|
||||
}
|
||||
|
||||
private static String longestCommonSubstring(String strA, String strB) {
|
||||
char[] chars_strA = strA.toCharArray();
|
||||
char[] chars_strB = strB.toCharArray();
|
||||
int m = chars_strA.length;
|
||||
int n = chars_strB.length;
|
||||
int[][] matrix = new int[m + 1][n + 1];
|
||||
for (int i = 1; i <= m; i++) {
|
||||
|
||||
for (int j = 1; j <= n; j++) {
|
||||
if (chars_strA[i - 1] == chars_strB[j - 1])
|
||||
matrix[i][j] = matrix[i - 1][j - 1] + 1;
|
||||
else
|
||||
matrix[i][j] = Math.max(matrix[i][j - 1], matrix[i - 1][j]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
char[] result = new char[matrix[m][n]];
|
||||
int currentIndex = result.length - 1;
|
||||
while (matrix[m][n] != 0) {
|
||||
if (matrix[m][n] == matrix[m][n - 1])
|
||||
n--;
|
||||
else if (matrix[m][n] == matrix[m - 1][n])
|
||||
m--;
|
||||
else {
|
||||
result[currentIndex] = chars_strA[m - 1];
|
||||
currentIndex--;
|
||||
n--;
|
||||
m--;
|
||||
}
|
||||
}
|
||||
return new String(result);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -1,8 +1,6 @@
|
|||
package com.ossean;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
|
|
@ -13,11 +11,12 @@ import org.springframework.context.ApplicationContext;
|
|||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.dao.DBSource;
|
||||
import com.ossean.dao.GatherDao;
|
||||
import com.ossean.dao.UpdateControlProjectsDao;
|
||||
import com.ossean.databaseSource.DBSource;
|
||||
import com.ossean.databaseSource.GatherDao;
|
||||
import com.ossean.databaseSource.UpdateControlProjectsDao;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.util.MergeProjectNew2;
|
||||
import com.ossean.util.MergeProjectNew;
|
||||
import com.ossean.util.MergeProjectsUtil;
|
||||
|
||||
@Component
|
||||
public class MergeProjects {
|
||||
|
|
@ -28,16 +27,19 @@ public class MergeProjects {
|
|||
private GatherDao gatherDao;
|
||||
@Resource
|
||||
private UpdateControlProjectsDao updateControlDao;
|
||||
@Qualifier("mergeProjectNew2")
|
||||
|
||||
@Qualifier("mergeProjectNew")
|
||||
@Autowired
|
||||
private MergeProjectNew2 mergeProjectNew2;
|
||||
private MergeProjectNew mergeProjectNew;
|
||||
|
||||
private int startId1;
|
||||
|
||||
private static String pointerTableName = "edd_pointers";
|
||||
private static String sourceTableName = "gather_projects";
|
||||
private static String targetTableName = "edd_relations";
|
||||
|
||||
private static String pointerTableName = TableName.pointerTableName;//记录去重进行到哪个地方了
|
||||
private static String sourceTableName = TableName.gatherProjectsTableName;//去重项目的来源，来自汇总程序生成的表
|
||||
private static String targetTableName = TableName.eddRelationTableName;//去重结果存储位置
|
||||
private static int batchSize = 500;
|
||||
public static Set<Integer> haveRemovedSet = new HashSet<Integer>();
|
||||
|
||||
//读指针
|
||||
public int readPointer(String table, String source, String target, int minId){
|
||||
int pointer = minId;
|
||||
|
|
@ -51,43 +53,61 @@ public class MergeProjects {
|
|||
}
|
||||
|
||||
public void start(){
|
||||
//int minId = 1;
|
||||
//int maxId = dbSource.getMaxId(sourceTableName);
|
||||
int count=0;
|
||||
count = readPointer(pointerTableName,sourceTableName,targetTableName, count);//指针表计量处理的项目数
|
||||
long start = System.currentTimeMillis();
|
||||
count = readPointer(pointerTableName, sourceTableName, targetTableName, count);
|
||||
|
||||
while(true){
|
||||
//取完别名
|
||||
List<GatherProjectsModel> gpmList = gatherDao.selectGPMListNew(sourceTableName,1,batchSize);
|
||||
if(gpmList.size()==0){
|
||||
logger.info("Duplicate remove:all projects have been done. Sleep 10 mins");
|
||||
logger.warn("deal with "+haveRemovedSet.size()+" projects cost: "+(float)(System.currentTimeMillis() - start)/60000+" minutes");
|
||||
//startId1 = readPointer(pointerTableName, sourceTableName, targetTableName, minId);
|
||||
// if(startId1 > maxId){
|
||||
//
|
||||
// try {
|
||||
// logger.info("没有数据了，休眠10min");
|
||||
// Thread.sleep(600*1000L);
|
||||
// continue;
|
||||
// } catch (InterruptedException e) {
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// }
|
||||
List<GatherProjectsModel> gpmList1 = gatherDao.selectGPMList(sourceTableName,batchSize);
|
||||
|
||||
if(gpmList1.size() == 0){
|
||||
try {
|
||||
Thread.sleep(600*1000);
|
||||
logger.info("no increase item to be handled! handle update items");
|
||||
List<GatherProjectsModel> updateProjectList = gatherDao.selectUpdateProjects(sourceTableName,batchSize);
|
||||
if(updateProjectList.size() == 0){
|
||||
logger.info("all update items have been solved. Sleep 10 mins");
|
||||
Thread.sleep(600*1000);
|
||||
continue;
|
||||
}
|
||||
else
|
||||
for(GatherProjectsModel model:updateProjectList){
|
||||
mergeProjectNew.handleNewProject(model, true);
|
||||
gatherDao.updateUpdateMark(sourceTableName, 1, model.getId());
|
||||
}
|
||||
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
else{
|
||||
for(GatherProjectsModel model:gpmList){
|
||||
if(haveRemovedSet.contains(model.getId()))
|
||||
continue;
|
||||
//logger.info("Duplicate remove:handling project " + model.getId());
|
||||
haveRemovedSet.addAll(mergeProjectNew2.handleNewProject(model));
|
||||
}
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, haveRemovedSet.size());
|
||||
if(haveRemovedSet.size()%10000==0)
|
||||
logger.warn("deal with:"+haveRemovedSet.size()+" projects cost: "+(float)(System.currentTimeMillis() - start)/60000+" minutes");
|
||||
}
|
||||
long start = System.currentTimeMillis();
|
||||
for(GatherProjectsModel model1:gpmList1){
|
||||
logger.info("handling project : " + model1.getId());
|
||||
mergeProjectNew.handleNewProject(model1, false);
|
||||
count = count+1;
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, count);
|
||||
gatherDao.updateUpdateMark(sourceTableName, 1, model1.getId()); //需要更新update_mark字段 因为按顺序取的时候可能取到1的
|
||||
}
|
||||
long end = System.currentTimeMillis();
|
||||
logger.warn("deal with: 500 projects cost: "+(float)(end - start)/60000+" minutes");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args){
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
MergeProjects Main = applicationContext.getBean(MergeProjects.class);
|
||||
GetSynonyms getSynonyms = (GetSynonyms) applicationContext.getBean("getSynonyms");
|
||||
getSynonyms.start();
|
||||
System.out.println("insertSynonyms thread start~");
|
||||
Main.start();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,341 @@
|
|||
package com.ossean;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.databaseSource.DBSource;
|
||||
import com.ossean.databaseSource.GatherDao;
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.util.MergeProjectsUtil;
|
||||
|
||||
@Component
|
||||
public class MergeProjectsByHomepageOnly {
|
||||
Logger logger = Logger.getLogger(this.getClass());
|
||||
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
|
||||
@Qualifier("mergeProjectsUtil")
|
||||
@Autowired
|
||||
private MergeProjectsUtil mergeProjectsUtil;
|
||||
|
||||
private static String pointerTableName = "edd_pointers";
|
||||
private static String sourceTableName = "gather_projects";
|
||||
private static String eddRelationTableName = "edd_relations_test";
|
||||
private static String targetTableName = "edd_relations_homepage_only1";
|
||||
|
||||
public int readPointer(String table, String source, String target){
|
||||
int pointer = 1;
|
||||
try {
|
||||
pointer = dbSource.getPointer(table, source, target);
|
||||
} catch(Exception e) {
|
||||
logger.info("No such pointer! Create one");
|
||||
dbSource.insertPointer(table, source, target, pointer);
|
||||
}
|
||||
return pointer;
|
||||
}
|
||||
|
||||
public void start(){
|
||||
while(true){
|
||||
int maxId = dbSource.getMaxId(sourceTableName);
|
||||
int startId1 = readPointer(pointerTableName, sourceTableName, targetTableName);
|
||||
if(startId1 > maxId){
|
||||
logger.info("finish program! HAHAHA");
|
||||
System.exit(0);
|
||||
}
|
||||
//获取当前的项目
|
||||
GatherProjectsModel project = gatherDao.selectGPMById(sourceTableName, startId1);
|
||||
if(project == null){
|
||||
//表示项目不存在
|
||||
logger.info("项目：" + startId1 + " 不存在");
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, startId1 + 1);
|
||||
continue;
|
||||
}
|
||||
String homepage = project.getHomepage();
|
||||
String source = project.getSource();
|
||||
//如果是null 不需要查找相同homepage的项目
|
||||
List<GatherProjectsModel> sameHomepageList = new ArrayList<GatherProjectsModel>();
|
||||
if(source=="apache" && (homepage == null||homepage.equals("")) ){
|
||||
homepage = project.getUrl();
|
||||
dbSource.updateApacheHomepageById(sourceTableName, homepage, project.getId());
|
||||
}
|
||||
if(homepage != null && !"".equals(homepage)){
|
||||
homepage = homepage.trim();
|
||||
//查找所有相同homepage字段的项目
|
||||
//判断homepage是否有http://或https://
|
||||
int index_http = homepage.indexOf("http://");
|
||||
int index_https = homepage.indexOf("https://");
|
||||
if(index_http == 0){
|
||||
homepage = homepage.substring(index_http + 7); //去除http://
|
||||
}else if(index_https == 0){
|
||||
homepage = homepage.substring(index_https + 8); //去除https://
|
||||
}else{
|
||||
//表示不是以http或https开头
|
||||
}
|
||||
|
||||
if((homepage.lastIndexOf("/") == homepage.length()-1) && (homepage.length() != 0)){
|
||||
//如果homepage的最后一个字符是/ 需要去除掉
|
||||
homepage = homepage.substring(0, homepage.length() - 1);
|
||||
}
|
||||
if(homepage.length() > 0){
|
||||
//为homepage添加http或https头
|
||||
String homepage1 = "http://" + homepage;
|
||||
String homepage2 = "https://" + homepage;
|
||||
String homepage3 = "http://" + homepage + "/";
|
||||
String homepage4 = "https://" + homepage + "/";
|
||||
|
||||
sameHomepageList = gatherDao.selectGPMBySameHomePage(sourceTableName,homepage, homepage1, homepage2, homepage3, homepage4, project.getId(), project.getSource());//认为一定是相同的项目
|
||||
}
|
||||
}
|
||||
|
||||
//处理sameHomepageList 如果edd_relations表中有包含对应id的记录 则将所有的id糅合在一起，删除以前的记录，添加新的edd_relation记录
|
||||
if(sameHomepageList.size() == 0){
|
||||
//表示没有找到对应homepage的项目
|
||||
//更新指针
|
||||
if(homepage == null){
|
||||
logger.info(project.getId() + "项目：" + project.getName() + " homepage为null");
|
||||
}else{
|
||||
logger.info(project.getId() + "项目：" + project.getName() + " 没有找到相同homepage的项目");
|
||||
}
|
||||
List<EddRelations> relationsList_now = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, project.getId());
|
||||
if(relationsList_now.size()==0){
|
||||
EddRelations relation = new EddRelations();
|
||||
String gather_projects_id = ","+project.getId()+",";
|
||||
relation.setGather_projects_ids(gather_projects_id);
|
||||
dbSource.insertEddRelations(eddRelationTableName, relation);
|
||||
}
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, startId1 + 1);
|
||||
continue;
|
||||
}
|
||||
//加入了项目本身
|
||||
sameHomepageList.add(project);
|
||||
|
||||
List<EddRelations> relationsList = new ArrayList<EddRelations>();
|
||||
EddRelations relation_model = new EddRelations();
|
||||
relation_model.setGather_projects_ids(","+project.getId()+",");
|
||||
relationsList.add(relation_model); //relationsList用于记录包含当前项目在内的
|
||||
|
||||
List<EddRelations> relationsList_matched = new ArrayList<EddRelations>();//匹配上的项目在edd_relations表中的记录
|
||||
for(int i = 0; i < sameHomepageList.size(); i++){
|
||||
GatherProjectsModel model_sameHP = sameHomepageList.get(i);
|
||||
//查找edd_relations表得到相应的记录
|
||||
List<EddRelations> eddRelationList = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, model_sameHP.getId());
|
||||
for(EddRelations relation:eddRelationList){
|
||||
relationsList.add(relation);
|
||||
relationsList_matched.add(relation);
|
||||
}
|
||||
if(eddRelationList.size() == 0){
|
||||
//表示之前没有匹配上项目的匹配记录 需要插入这条记录 eg: ,12,
|
||||
EddRelations relation = new EddRelations();
|
||||
relation.setGather_projects_ids(","+model_sameHP.getId()+",");
|
||||
relationsList.add(relation); //relationsList用于记录包含当前项目在内的
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//根据新的匹配结果形成一个新的EddRelation对象
|
||||
// EddRelations new_relation = new EddRelations();
|
||||
// String new_gather_projects_ids = ",";
|
||||
// for(GatherProjectsModel model:sameHomepageList){
|
||||
// new_gather_projects_ids = new_gather_projects_ids + model.getId() + ",";
|
||||
// }
|
||||
// new_relation.setGather_projects_ids(new_gather_projects_ids);
|
||||
// relationsList.add(new_relation);
|
||||
// //倘若都是新项目，则resultInEddRelation为空！longestRelation为空
|
||||
// List<EddRelations> resultInEddRelation = new ArrayList<EddRelations>();
|
||||
// for(GatherProjectsModel model:sameHomepageList){
|
||||
// List<EddRelations> relationList = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, model.getId());
|
||||
// for(EddRelations relation:relationList){
|
||||
// resultInEddRelation.add(relation);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// //判断resultInEddRelation中最长的gather_projects_ids是不是包含了其他所有的结果项目 如果不是则需要将所有的项目id统计到一起并删除原来的所有结果
|
||||
// //找到resultInEddRelation中最长的gather_projects_ids对应的对象
|
||||
// EddRelations longestRelation = findLongestGatherProjectsIds(resultInEddRelation);
|
||||
//
|
||||
// //将new_relation插入resultInEddRelation中
|
||||
// resultInEddRelation.add(new_relation);
|
||||
//
|
||||
// //relationsList_now表示目前正在处理的项目在edd_relations表中的记录 需要在插入新的edd_relations对象前查找
|
||||
// List<EddRelations> relationsList_now = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, project.getId());
|
||||
//
|
||||
// //判断最长的gather_projects_ids中是不是包括了所有其他eddrelation中匹配到的id
|
||||
// //倘若都是新项目，则longestRelation为空！
|
||||
// boolean hasOthers = containOtherRelations(longestRelation, resultInEddRelation);
|
||||
//
|
||||
// //如果hasOthers为true 表示确实包含所有其他的relation（包括新加入的relation） 因此不用重新组合插入新的relation
|
||||
// if(hasOthers == true){
|
||||
// //删除除longestRelation以外所有数据库中的relation 其中new_relation对应的id为默认值0 可以一并删除（数据库中没有 但删除操作可以继续）
|
||||
// for(EddRelations relation:resultInEddRelation){
|
||||
// if(relation.getId() != longestRelation.getId()){
|
||||
// dbSource.deleteEddRelationsItem(eddRelationTableName, relation.getId());
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
//如果hasOthers为false 表示存在edd_relations对象中的项目id不包含在最长的gather_projects_ids中 需要重新组合所有id 并删除原来的edd_relations插入新的edd_relations
|
||||
List<EddRelations> relationsList_now = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, project.getId());
|
||||
String new_relation_string = mergeProjectsUtil.hasDiffProjFromSameCommunity(relationsList);
|
||||
//判断所有id是不是都来自于不同的社区 如果是则插入newEddRelation 如果不是 则不进行操作
|
||||
if(new_relation_string == null){
|
||||
//表示有来自同一社区的项目，则不进行后续操作
|
||||
if(relationsList_now.size() == 0){
|
||||
//表示不是更新的项目 需要插入当前项目本身的记录
|
||||
EddRelations relation = new EddRelations();
|
||||
String gather_projects_id = ","+project.getId()+",";
|
||||
relation.setGather_projects_ids(gather_projects_id);
|
||||
dbSource.insertEddRelations(eddRelationTableName, relation);
|
||||
}
|
||||
}
|
||||
else{
|
||||
//表示匹配结果中不存在同一个社区不同的多个项目匹配到一起
|
||||
EddRelations eddRelation = new EddRelations();
|
||||
eddRelation.setGather_projects_ids(new_relation_string);
|
||||
dbSource.insertEddRelations(eddRelationTableName, eddRelation);//插入最新的结果集
|
||||
/**
|
||||
* 删除关联项目在edd_relations表中对应的记录
|
||||
*/
|
||||
if(relationsList_matched.size() !=0){
|
||||
for(EddRelations relation:relationsList_matched)
|
||||
deleteExistEddRelations(relation);
|
||||
}/**
|
||||
* 如果是更新项目 需要查看edd_relations表中是否有该记录 如果有要删除掉
|
||||
*/
|
||||
if(relationsList_now.size() !=0){
|
||||
for(EddRelations relation:relationsList_now)
|
||||
deleteExistEddRelations(relation);
|
||||
}
|
||||
}
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, startId1 + 1);
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
public void deleteExistEddRelations(EddRelations relation){
|
||||
//删除之前对应的relation记录
|
||||
int edd_relation_id = relation.getId();
|
||||
dbSource.deleteEddRelationsItem(eddRelationTableName, edd_relation_id);
|
||||
}
|
||||
|
||||
|
||||
private EddRelations findLongestGatherProjectsIds(List<EddRelations> eddRelationList){
|
||||
EddRelations result = new EddRelations();
|
||||
int maxLength = 0;
|
||||
for(EddRelations relation:eddRelationList){
|
||||
String gather_projects_ids = relation.getGather_projects_ids();
|
||||
int length = gather_projects_ids.split(",").length-1;
|
||||
if(length > maxLength){
|
||||
maxLength = length;
|
||||
result = relation;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
private boolean containOtherRelations(EddRelations longestRelation, List<EddRelations> relationList){
|
||||
boolean result = true;
|
||||
//System.out.println(longestRelation);
|
||||
if(longestRelation.getGather_projects_ids() == null)
|
||||
return false;
|
||||
for(EddRelations relation:relationList){
|
||||
if(containRelation(longestRelation, relation) == false){
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean containRelation(EddRelations longestRelation, EddRelations relation){
|
||||
boolean result = false;
|
||||
String longestIds = longestRelation.getGather_projects_ids();
|
||||
String[] longestIdArray = longestIds.split(",");
|
||||
//去除第一个空串元素
|
||||
longestIdArray = removeFirstElement(longestIdArray);
|
||||
String[] relationIdArray = relation.getGather_projects_ids().split(",");
|
||||
//去除第一个空串元素
|
||||
relationIdArray = removeFirstElement(relationIdArray);
|
||||
//查看array2中的元素在array1中是否都存在
|
||||
result = containArray(longestIdArray, relationIdArray);
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean containArray(String[] array1, String[] array2){
|
||||
boolean result = true;
|
||||
for(String s2:array2){
|
||||
int i;
|
||||
for(i = 0; i < array1.length; i++){
|
||||
String s1 = array1[i];
|
||||
if(s1.equals(s2)){
|
||||
break; //表示存在结果 匹配下一个string
|
||||
}
|
||||
}
|
||||
if(i == array1.length){
|
||||
//表示匹配到最后都没有匹配到合适的结果
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
//重新组合List<EddRelations>中EddRelation对象对应的gather_projects_ids对应的id数据
|
||||
private EddRelations combineNewEddRelation(List<EddRelations> eddRelationList){
|
||||
EddRelations result = new EddRelations();
|
||||
String new_gather_projects_ids = ",";
|
||||
Map<String, Integer> idMap = new HashMap<String, Integer>();
|
||||
for(EddRelations relation:eddRelationList){
|
||||
String gather_projects_ids = relation.getGather_projects_ids();
|
||||
String[] idArray = gather_projects_ids.split(",");
|
||||
for(String id:idArray){
|
||||
if(!"".equals(id))
|
||||
idMap.put(id, 1);
|
||||
}
|
||||
}
|
||||
//根据map中的key形成new_gather_projects_ids
|
||||
Set<String> keys = idMap.keySet();
|
||||
Iterator it = keys.iterator();
|
||||
while(it.hasNext()){
|
||||
String key = it.next().toString();
|
||||
new_gather_projects_ids = new_gather_projects_ids + key + ",";
|
||||
}
|
||||
result.setGather_projects_ids(new_gather_projects_ids);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
private String[] removeFirstElement(String[] array){
|
||||
String[] result = new String[array.length-1];
|
||||
for(int i = 1; i < array.length; i++){
|
||||
result[i-1] = array[i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
MergeProjectsByHomepageOnly Main = applicationContext.getBean(MergeProjectsByHomepageOnly.class);
|
||||
Main.start();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
package com.ossean;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.databaseSource.DBSource;
|
||||
import com.ossean.databaseSource.GatherDao;
|
||||
import com.ossean.databaseSource.UpdateControlProjectsDao;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.util.MergeProjectNew;
|
||||
import com.ossean.util.MergeProjectsUtil;
|
||||
|
||||
@Component
|
||||
public class MergeProjectsOld {
|
||||
Logger logger = Logger.getLogger(this.getClass());
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
@Resource
|
||||
private UpdateControlProjectsDao updateControlDao;
|
||||
|
||||
@Qualifier("mergeProjectNew")
|
||||
@Autowired
|
||||
private MergeProjectNew mergeProjectNew;
|
||||
//private MergeProjectsUtil mergeProjectsUtil;
|
||||
|
||||
private int startId1;
|
||||
|
||||
private static String pointerTableName = "edd_pointers_test";
|
||||
private static String sourceTableName = "gather_projects_test";
|
||||
private static String targetTableName = "edd_relations_test1";
|
||||
|
||||
private static int batchSize = 500;
|
||||
|
||||
//读指针
|
||||
public int readPointer(String table, String source, String target, int minId){
|
||||
int pointer = minId;
|
||||
try {
|
||||
pointer = dbSource.getPointer(table, source, target);
|
||||
} catch(Exception e) {
|
||||
logger.info("No such pointer! Create one");
|
||||
dbSource.insertPointer(table, source, target, pointer);
|
||||
}
|
||||
return pointer;
|
||||
}
|
||||
|
||||
public void start(String edd_relations_pointer){
|
||||
int minId = 1;
|
||||
int maxId = Integer.MAX_VALUE;
|
||||
if(edd_relations_pointer.equals("edd_relations")){
|
||||
minId = 1;
|
||||
maxId = 200000;
|
||||
}else if(edd_relations_pointer.equals("edd_relations_pointer2")){
|
||||
minId = 200001;
|
||||
maxId = 400000;
|
||||
}else if(edd_relations_pointer.equals("edd_relations_pointer3")){
|
||||
minId = 400001;
|
||||
maxId = 600000;
|
||||
}else if(edd_relations_pointer.equals("edd_relations_pointer4")){
|
||||
minId = 600001;
|
||||
maxId = 800000;
|
||||
}else if(edd_relations_pointer.equals("edd_relations_pointer_others")){
|
||||
minId = 800001;
|
||||
}else{}
|
||||
while(true){
|
||||
startId1 = readPointer(pointerTableName, sourceTableName, edd_relations_pointer, minId);
|
||||
List<GatherProjectsModel> gpmList1 = gatherDao.selectGPMList(sourceTableName, batchSize);
|
||||
if(gpmList1.size() == 0){
|
||||
try {
|
||||
logger.info("no increase item to be handled! handle update items");
|
||||
//查看有没有更新的记录需要处理 通过gather_projects表中的update_mark字段进行判断 为1表示更新了数据
|
||||
// List<UpdateControlProjects> updateProjectList = updateControlDao.findAllItems(updateControlProjectsTableName);
|
||||
//不考虑更新了 湛云没有使用update_control_projects表控制更新
|
||||
// if(updateProjectList.size() != 0){
|
||||
// //表示有需要更新的项目
|
||||
// for(UpdateControlProjects updateProject:updateProjectList){
|
||||
// //重新处理更新项目
|
||||
// GatherProjectsModel model = gatherDao.selectGPMById(sourceTableName, updateProject.getId());
|
||||
// mergeProjectsUtil.handleNewProject(model, true);
|
||||
// updateControlDao.deleteOneItem(updateControlProjectsTableName, updateProject.getId());
|
||||
// }
|
||||
// }
|
||||
List<GatherProjectsModel> updateProjectList = gatherDao.selectUpdateProjects(sourceTableName,batchSize);
|
||||
for(GatherProjectsModel model:updateProjectList){
|
||||
mergeProjectNew.handleNewProject(model, true);
|
||||
gatherDao.updateUpdateMark(sourceTableName, 1, model.getId());
|
||||
}
|
||||
logger.info("all update items have been solved. Sleep 1h");
|
||||
Thread.sleep(3600*1000);
|
||||
continue;
|
||||
} catch (InterruptedException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
for(GatherProjectsModel model1:gpmList1){
|
||||
logger.info("handling project : " + model1.getId());
|
||||
// List<GatherProjectsModel> sameProjects = gatherDao.selectGPMByUrlMD5(sourceTableName, model1.getHomepageMD5());
|
||||
|
||||
mergeProjectNew.handleNewProject(model1, false);
|
||||
|
||||
//如果是对更新表中的数据进行操作 则不更新指针
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, edd_relations_pointer, model1.getId() + 1);
|
||||
gatherDao.updateUpdateMark(sourceTableName, 1, model1.getId()); //需要更新update_mark字段 因为按顺序取的时候可能取到1的
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static void main(String[] args){
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
MergeProjectsOld Main = applicationContext.getBean(MergeProjectsOld.class);
|
||||
Main.start(args[0]);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
package com.ossean;
|
||||
|
||||
public class TableName {
|
||||
|
||||
public static String synonymsTableName = "synonymmings_2017";
|
||||
public static String pointerTableName = "edd_pointers";
|
||||
public static String gatherProjectsTableName = "gather_projects";
|
||||
public static String taggingsTableName = "taggings";
|
||||
public static String tagsTableName = "tags";
|
||||
public static String eddRelationTableName = "edd_relations";
|
||||
public static String openSourceProjectsTableName = "open_source_projects_17";
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
package com.ossean;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.databaseSource.GatherDao;
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.util.MergeProjectsUtil;
|
||||
import com.ossean.util.StringHandler_ProjectName;
|
||||
import com.ossean.util.TransferProjectsUtil;
|
||||
|
||||
|
||||
@Component
|
||||
public class Test {
|
||||
|
||||
@Resource
|
||||
private TransferProjectsUtil util;
|
||||
@Resource
|
||||
private MergeProjectsUtil mergeUtil;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
|
||||
private MergeProjectsByHomepageOnly mergeHomepage;
|
||||
|
||||
|
||||
|
||||
String synonyms = "<maven>,<maven>,<hello>,<shit>,<finally>,<shit>,";
|
||||
String synonyms2 = "";
|
||||
|
||||
String name = " mozillasolr ";
|
||||
String description = " shit is very good";
|
||||
String homepage = "http://www.7-zip.org/";
|
||||
|
||||
String tag1 = "<1>,<2>,<3>,<4>,<5>";
|
||||
String tag2 = "<3>,<4>";
|
||||
|
||||
String source1 = "sourceforge";
|
||||
String source2 = "sourceforge";
|
||||
|
||||
|
||||
public void start(){
|
||||
//测试combineNewEddRelations函数
|
||||
EddRelations relation1 = new EddRelations();
|
||||
EddRelations relation2 = new EddRelations();
|
||||
EddRelations relation3 = new EddRelations();
|
||||
String gather_projects_ids1 = ",1,2,3,4,";
|
||||
String gather_projects_ids2 = ",1,";
|
||||
String gather_projects_ids3 = ",3,4,5,6,";
|
||||
|
||||
String[] array1 = gather_projects_ids1.split(",");
|
||||
String[] array2 = gather_projects_ids2.split(",");
|
||||
|
||||
// boolean result = containArray(array1, array2);
|
||||
|
||||
relation1.setGather_projects_ids(gather_projects_ids1);
|
||||
relation2.setGather_projects_ids(gather_projects_ids2);
|
||||
relation3.setGather_projects_ids(gather_projects_ids3);
|
||||
|
||||
List<EddRelations> relationList = new ArrayList<EddRelations>();
|
||||
relationList.add(relation2);
|
||||
relationList.add(relation3);
|
||||
relationList.add(relation1);
|
||||
|
||||
EddRelations result = findLongestGatherProjectsIds(relationList);
|
||||
|
||||
// EddRelations result = combineNewEddRelation(relationList);
|
||||
// System.out.println(result.getGather_projects_ids());
|
||||
}
|
||||
|
||||
|
||||
private EddRelations findLongestGatherProjectsIds(List<EddRelations> eddRelationList){
|
||||
EddRelations result = new EddRelations();
|
||||
int maxLength = 0;
|
||||
for(EddRelations relation:eddRelationList){
|
||||
String gather_projects_ids = relation.getGather_projects_ids();
|
||||
int length = gather_projects_ids.split(",").length-1;
|
||||
if(length > maxLength){
|
||||
maxLength = length;
|
||||
result = relation;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
Test Main = applicationContext.getBean(Test.class);
|
||||
Main.start();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -11,12 +11,10 @@ import org.springframework.context.ApplicationContext;
|
|||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.dao.DBDest;
|
||||
import com.ossean.dao.DBSource;
|
||||
import com.ossean.dao.TransferPrjDao;
|
||||
import com.ossean.databaseDest.DBDest;
|
||||
import com.ossean.databaseSource.DBSource;
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.util.TransferProjectsUtil;
|
||||
import com.ossean.util.TransferProjectsUtil2;
|
||||
|
||||
|
||||
@Component
|
||||
|
|
@ -27,16 +25,14 @@ public class TransferProjects {
|
|||
private DBSource dbSource;
|
||||
@Resource
|
||||
private DBDest dbDest;
|
||||
@Resource
|
||||
private TransferPrjDao transferPrjDao;
|
||||
@Qualifier("transferProjectsUtil2")
|
||||
@Qualifier("transferProjectsUtil")
|
||||
@Autowired
|
||||
private TransferProjectsUtil2 transferProjectsUtil2;
|
||||
private TransferProjectsUtil transferProjectsUtil;
|
||||
|
||||
|
||||
private String pointerTableName = TableName.pointerTableName;
|
||||
private String sourceTableName = TableName.eddRelationTableName;
|
||||
private String targetTableName = TableName.openSourceProjectsTableName;
|
||||
private String pointerTableName = "edd_pointers";
|
||||
private String sourceTableName = "edd_relations";
|
||||
private String targetTableName = "open_source_projects";
|
||||
|
||||
private int batchSize = 100;
|
||||
|
||||
|
|
@ -65,7 +61,7 @@ public class TransferProjects {
|
|||
public void start(){
|
||||
while(true){
|
||||
int startId = readPointer(pointerTableName, sourceTableName, targetTableName);
|
||||
List<EddRelations> eddRelationList = transferPrjDao.getEddRelationList(sourceTableName, startId, batchSize);
|
||||
List<EddRelations> eddRelationList = dbSource.getEddRelationList(sourceTableName, startId, batchSize);
|
||||
if(eddRelationList.size() == 0){
|
||||
try {
|
||||
logger.info("没有数据了，休眠10min");
|
||||
|
|
@ -76,8 +72,7 @@ public class TransferProjects {
|
|||
}
|
||||
}
|
||||
for(EddRelations relation:eddRelationList){
|
||||
transferProjectsUtil2.handleOneRelation(relation);
|
||||
dbSource.updatePointer(pointerTableName, sourceTableName, targetTableName, relation.getId()+1);
|
||||
transferProjectsUtil.handleOneRelation(relation);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,47 +0,0 @@
|
|||
package com.ossean.dao;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
|
||||
public interface GatherDao {
|
||||
|
||||
//tag表Insert操作
|
||||
@Insert("insert ignore into ${table} (name) values (#{name})")
|
||||
public void insertTag(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
//根据tag的name查找tag的id
|
||||
@Select("select id from ${table} where name=#{name}")
|
||||
public int selectTagIdByName(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
//根据urlMD5码找到GatherProjectModel对象
|
||||
@Select("select * from ${table} where urlMD5=#{urlMD5}")
|
||||
public GatherProjectsModel selectGPMByUrlMD5(@Param("table") String table, @Param("urlMD5") String urlMD5);
|
||||
|
||||
//根据update_mark，分别取不同状态的项目列表
|
||||
@Select("select * from ${table} where update_mark = #{mark} limit #{size}")
|
||||
public List<GatherProjectsModel> selectGPMListNew(@Param("table") String table,@Param("mark") int mark,@Param("size") int size);
|
||||
|
||||
//根据id取GatherProjectsModel对象
|
||||
@Select("select * from ${table} where id=#{id}")
|
||||
public GatherProjectsModel selectGPMById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//查找相同homepage的项目
|
||||
@Select("select * from ${table} where (homepage=#{homepage} or homepage=#{homepage1} or homepage=#{homepage2} or homepage=#{homepage3} or homepage=#{homepage4}) and update_mark = #{mark} and id!=#{id}")
|
||||
public List<GatherProjectsModel> selectGPMBySameHomePage(@Param("table") String table, @Param("homepage") String homepage, @Param("homepage1") String homepage1, @Param("homepage2") String homepage2, @Param("homepage3") String homepage3, @Param("homepage4") String homepage4, @Param("id") int id, @Param("mark") int mark);
|
||||
|
||||
//查找相同name的项目
|
||||
@Select("select * from ${table} where update_mark = #{mark} and name=#{name} and id!=#{id}")
|
||||
public List<GatherProjectsModel> selectGPMBySameName(@Param("table") String table, @Param("name") String name, @Param("id") int id,@Param("mark") int mark);
|
||||
|
||||
//更新update_mark字段
|
||||
@Select("update ${table} set update_mark=#{update_mark} where id=#{id}")
|
||||
public void updateMark(@Param("table") String table, @Param("update_mark") int update_mark, @Param("id") int id);
|
||||
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
package com.ossean.dao;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.model.OpenSourceProject;
|
||||
|
||||
public interface TransferPrjDao {
|
||||
//获取edd_relations表中数据
|
||||
@Select("select * from ${table} where id>=#{start} limit #{size}")
|
||||
public List<EddRelations> getEddRelationList(@Param("table") String table, @Param("start") int start, @Param("size") int size);
|
||||
|
||||
//更新edd_relations表的osp_id字段
|
||||
@Update("update ${table} set osp_id=#{osp_id} where id=#{id}")
|
||||
public void updateEddRelationOspId(@Param("table") String table, @Param("id") int id, @Param("osp_id") int osp_id);
|
||||
|
||||
//更新edd_relations表的gather_projects_ids字段
|
||||
@Update("update ${table} set gather_projects_ids=#{gather_projects_ids} where id=#{id}")
|
||||
public void updateEddRelationGatherProjectsIds(@Param("table") String table, @Param("id") int id, @Param("gather_projects_ids") String gather_projects_ids);
|
||||
|
||||
|
||||
//读取open_source_projects表中的记录
|
||||
@Select("select * from ${table} where id>=#{startId} limit #{size}")
|
||||
public List<OpenSourceProject> getOpenSourceProjectList(@Param("table") String table, @Param("startId") int startId, @Param("size") int size);
|
||||
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
package com.ossean.dao2;
|
||||
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
import com.ossean.model.OpenhubProject;
|
||||
import com.ossean.model.OschinaProject;
|
||||
import com.ossean.model.SourceForgeProject;
|
||||
|
||||
|
||||
public interface PlatformProjectDao {
|
||||
|
||||
@Select("select name,rate_num,user_num from openhub_project where url = #{url} group by url_md5 order by extracted_time desc")
|
||||
public OpenhubProject getOpenHubPrjByUrl( @Param("url") String url);
|
||||
|
||||
@Select("select name,download_num,favor_num from sourceforge_project where url = #{url} group by url_md5 order by extracted_time desc")
|
||||
public SourceForgeProject getSourceForgePrjByUrl( @Param("url") String url);//oschina_project
|
||||
|
||||
@Select("select name,star_num,user_num from sourceforge_project where url = #{url} group by url_md5 order by extracted_time desc")
|
||||
public OschinaProject getOschinaPrjByUrl(@Param("url") String url);
|
||||
|
||||
}
|
||||
|
|
@ -1,61 +1,57 @@
|
|||
package com.ossean.dao;
|
||||
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
import com.ossean.model.OpenSourceProject;
|
||||
import com.ossean.model.Taggings;
|
||||
|
||||
|
||||
|
||||
public interface DBDest {
|
||||
|
||||
//删除open_source_projects表中对应id的数据
|
||||
@Delete("delete from ${table} where id=#{id}")
|
||||
public void deleteOpenSourceProjectsItem(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//查找open_source_projects表对应id的记录
|
||||
@Select("select * from ${table} where id=#{id}")
|
||||
public OpenSourceProject selectOpenSourceProjectsItem(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
|
||||
/**
|
||||
* 下面是transferProjects程序的函数
|
||||
*/
|
||||
//向open_source_projects表中插入对象数据
|
||||
@Insert("insert into ${table} (id,name,description,"
|
||||
+ "url,url_md5,language,category,homepage,license,"
|
||||
+ "source,created_time,updated_time,extracted_time,"
|
||||
+ "tags,tags_for_search,synonyms,update_mark) values (#{model.id},#{model.name},#{model.description},"
|
||||
+ "#{model.url},#{model.url_md5},#{model.language},"
|
||||
+ "#{model.category},#{model.homepage},#{model.license},#{model.source},"
|
||||
+ "#{model.created_time},#{model.updated_time},#{model.extracted_time},#{model.tags},"
|
||||
+ "#{model.tags_for_search},#{model.synonyms},#{model.update_mark})")
|
||||
public void insertOsp(@Param("table") String table, @Param("model") OpenSourceProject model);
|
||||
|
||||
//查找刚刚插入open_source_projects表中的记录id
|
||||
@Select("select id from ${table} where name=#{model.name} order by id desc limit 1")
|
||||
public int getAutoIncrementOspId(@Param("table") String table, @Param("model") OpenSourceProject model);
|
||||
|
||||
//插入tag
|
||||
@Insert("insert ignore into ${table} (name) values (#{name})")
|
||||
public void insertTag(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
//根据tag name查找id
|
||||
@Select("select id from ${table} where name=#{name}")
|
||||
public int selectTagIdByName(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
//插入tagging
|
||||
@Insert("insert ignore into ${table} (tag_id,taggable_id,taggable_type,context,created_at,disagree_num) values (#{model.tag_id},#{model.taggable_id},#{model.taggable_type},#{model.context},#{model.created_at},#{model.disagree_num})")
|
||||
public void insertTagging(@Param("table") String table, @Param("model") Taggings model);
|
||||
|
||||
//删除对应ospId的匹配结果
|
||||
@Delete("delete from ${table} where osp_id = ${ospId}")
|
||||
public void deleteMatchResult(@Param("ospId") int ospId, @Param("table") String table);
|
||||
|
||||
//有相同别名的项目数量
|
||||
@Select("select count(*) from ${table} where synonyms like '%,${name},%' ")
|
||||
public int selectPrjNumFromOSP(@Param("table") String table, @Param("name") String name);
|
||||
package com.ossean.databaseDest;
|
||||
|
||||
import org.apache.ibatis.annotations.Delete;
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
import com.ossean.model.OpenSourceProject;
|
||||
import com.ossean.model.Taggings;
|
||||
|
||||
|
||||
|
||||
public interface DBDest {
|
||||
|
||||
//删除open_source_projects表中对应id的数据
|
||||
@Delete("delete from ${table} where id=#{id}")
|
||||
public void deleteOpenSourceProjectsItem(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//查找open_source_projects表对应id的记录
|
||||
@Select("select * from ${table} where id=#{id}")
|
||||
public OpenSourceProject selectOpenSourceProjectsItem(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
|
||||
/**
|
||||
* 下面是transferProjects程序的函数
|
||||
*/
|
||||
//向open_source_projects表中插入对象数据
|
||||
@Insert("insert into ${table} (id,name,description,"
|
||||
+ "url,url_md5,language,category,homepage,license,"
|
||||
+ "source,created_time,updated_time,extracted_time,"
|
||||
+ "tags,tags_for_search,synonyms,update_mark) values (#{model.id},#{model.name},#{model.description},"
|
||||
+ "#{model.url},#{model.url_md5},#{model.language},"
|
||||
+ "#{model.category},#{model.homepage},#{model.license},#{model.source},"
|
||||
+ "#{model.created_time},#{model.updated_time},#{model.extracted_time},#{model.tags},"
|
||||
+ "#{model.tags_for_search},#{model.synonyms},#{model.update_mark})")
|
||||
public void insertOsp(@Param("table") String table, @Param("model") OpenSourceProject model);
|
||||
|
||||
//查找刚刚插入open_source_projects表中的记录id
|
||||
@Select("select id from ${table} where name=#{model.name} order by id desc limit 1")
|
||||
public int getAutoIncrementOspId(@Param("table") String table, @Param("model") OpenSourceProject model);
|
||||
|
||||
//插入tag
|
||||
@Insert("insert ignore into ${table} (name) values (#{name})")
|
||||
public void insertTag(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
//根据tag name查找id
|
||||
@Select("select id from ${table} where name=#{name}")
|
||||
public int selectTagIdByName(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
//插入tagging
|
||||
@Insert("insert ignore into ${table} (tag_id,taggable_id,taggable_type,context,created_at,disagree_num) values (#{model.tag_id},#{model.taggable_id},#{model.taggable_type},#{model.context},#{model.created_at},#{model.disagree_num})")
|
||||
public void insertTagging(@Param("table") String table, @Param("model") Taggings model);
|
||||
|
||||
//删除对应ospId的匹配结果
|
||||
@Delete("delete from ${table} where osp_id = ${ospId}")
|
||||
public void deleteMatchResult(@Param("ospId") int ospId, @Param("table") String table);
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.ossean.dao;
|
||||
package com.ossean.databaseSource;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -19,7 +19,9 @@ import com.ossean.model.Tags;
|
|||
import com.ossean.model.UpdateControlProjects;
|
||||
|
||||
public interface DBSource {
|
||||
//查看指针edd_pointers表，查看处理数量
|
||||
|
||||
|
||||
|
||||
@Select("select Pointer from ${table} where SourceTableName=#{SourceTableName} and TargetTableName=#{TargetTableName}")
|
||||
public int getPointer(@Param("table") String table, @Param("SourceTableName") String SourceTableName, @Param("TargetTableName") String TargetTableName);
|
||||
|
||||
|
|
@ -31,10 +33,22 @@ public interface DBSource {
|
|||
@Select("select max(id) from ${table}")
|
||||
public int getMaxId(@Param("table") String table);
|
||||
|
||||
//读取item_tag_relation表对应id后面的数据
|
||||
@Select("select * from ${table} where id>#{id}")
|
||||
public List<Taggings> getRemainItems(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//根据标签名读取标签id
|
||||
@Select("select * from ${table} where name=#{name}")
|
||||
public Tags getTagByName(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
//根据标签id读取Tag对象
|
||||
@Select("select * from ${table} where id=#{id}")
|
||||
public Tags getTagById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//读取对应表中项目总数
|
||||
@Select("select count(*) from ${table} where taggable_type=#{type}")
|
||||
public int getItemNumber(@Param("table") String table, @Param("type") String type);
|
||||
|
||||
//从taggings表中读取对应gather_projects_id的数据
|
||||
@Select("select * from ${table} where taggable_id=#{id} and taggable_type='OpenSourceProject'")
|
||||
public List<Taggings> getTaggingsListByTaggableId(@Param("table") String table, @Param("id") int id);
|
||||
|
|
@ -47,6 +61,7 @@ public interface DBSource {
|
|||
@Select("select count(*) from ${table} where (tag_id=#{tagId1} or tag_id=#{tagId2}) and taggable_type='OpenSourceProject'")
|
||||
public int getNumOfTagIdAndOspByTwoTagIds(@Param("table") String table, @Param("tagId1") int tagId1, @Param("tagId2") int tagId2);
|
||||
|
||||
|
||||
//重载的上面一个函数
|
||||
@Select("select count(*) from ${table} where tag_id=#{tagId1} and taggable_type='OpenSourceProject'")
|
||||
public int getNumOfTagIdAndOsp(@Param("table") String table, @Param("tagId1") int tagId);
|
||||
|
|
@ -93,53 +108,24 @@ public interface DBSource {
|
|||
@Insert("insert ignore into ${table} (name) values (#{model.name})")
|
||||
public void insertSynonym(@Param("table") String table, @Param("model") Synonyms model);
|
||||
|
||||
//插入别名到synonym表
|
||||
@Insert("insert ignore into ${table} (prjId,synonyms,flag) VALUES (#{prjId},#{synonyms},#{flag})")
|
||||
public void insertSynonymsNew(@Param("table") String table, @Param("prjId") int prjId,@Param("synonyms") String synonyms,@Param("flag") int flag);
|
||||
|
||||
//根据同义词名查找同义词对象
|
||||
@Select("select * from ${table} where name=#{name}")
|
||||
public Synonyms getSynonymByName(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
@Update("update ${table} set flag=#{flag} where prjId=#{id}")
|
||||
public void updateSynonyms(@Param("table") String table, @Param("id") int id,@Param("flag") int flag);
|
||||
|
||||
//插入Synonymmings对象
|
||||
@Insert("insert ignore into ${table} (synonym_id,synonymming_id,flag) values (#{model.synonym_id},#{model.synonymming_id},#{model.flag})")
|
||||
@Insert("insert ignore into ${table} (synonym_id,synonymming_id) values (#{model.synonym_id},#{model.synonymming_id})")
|
||||
public void insertSynonymming(@Param("table") String table, @Param("model") Synonymmings model);
|
||||
|
||||
//取出synonymmings对象
|
||||
@Select("select * from ${table} where synonym_id=#{synonym_id} and synonymming_id=#{synonymming_id}")
|
||||
public Synonymmings selectSynonymming(@Param("table") String table, @Param("synonym_id") int synonym_id, @Param("synonymming_id") int synonymming_id);
|
||||
|
||||
|
||||
//根据synonym_id找到所有synonymmings对象
|
||||
@Select("select * from ${table} where synonym_id=#{synonym_id}")
|
||||
public List<Synonymmings> selectSynonymmingListBySynonymId(@Param("table") String table, @Param("synonym_id") int synonym_id);
|
||||
|
||||
//读取synonymmings表
|
||||
@Select("select synonym_id from ${table} where synonymming_id=#{synonymming_id} and flag=1")
|
||||
public List<Integer> getSynonymmingList(@Param("table") String table, @Param("synonymming_id") int synonymming_id);
|
||||
|
||||
//读取synonym对象
|
||||
@Select("select * from ${table} where id=#{id}")
|
||||
public Synonyms getSynonymById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//读取对应于项目ID的synonym对象
|
||||
@Select("select synonyms from ${table} where prjId=#{id} and flag=1;")
|
||||
public List<String> getSynonymByPrjId(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//transfer中:读取对应于项目ID的synonym对象，只需确定的flag=1别名
|
||||
@Select("select synonyms from ${table} where prjId=#{id} and flag=1;")
|
||||
public List<String> getSynonymByPrjIdForTransfer(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//读取对应于项目ID的synonym对象
|
||||
@Select("select prjId from ${table} where synonyms=#{synonyms}")
|
||||
public List<Integer> getSynonymBySynonyms(@Param("table") String table, @Param("synonyms") String synonyms);
|
||||
|
||||
//读取synonym对象
|
||||
@Select("select name from ${table} where id=#{id}")
|
||||
public String getSynonymNameById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//根据url检测是否存在匹配结果
|
||||
@Select("select * from ${table} where url like '%|:|${url}' escape '/' or url like '%|:|${url}|,|%' escape '/'")
|
||||
public EddProjects selectEddProjectsByUrl(@Param("table") String table, @Param("url") String url);
|
||||
|
|
@ -147,6 +133,15 @@ public interface DBSource {
|
|||
@Select("select * from ${table} where id=#{id}")
|
||||
public EddProjects selectEddProjectsById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//根据删除表中对应的数据
|
||||
@Delete("delete from ${table} where id=#{id}")
|
||||
public void deleteItemById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
|
||||
//根据id获取EddProjects对象信息
|
||||
@Select("select * from ${table} where id=#{id}")
|
||||
public EddProjects getEddProjectsById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//获取edd_relations表中记录相应gather_projects_id的记录
|
||||
@Select("select * from ${table} where gather_projects_ids like '%,${gather_projects_id},%'")
|
||||
public List<EddRelations> getEddRelationsByGatherProjectsId(@Param("table") String table, @Param("gather_projects_id") int gather_projects_id);
|
||||
|
|
@ -156,10 +151,44 @@ public interface DBSource {
|
|||
public void deleteEddRelationsItem(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//向edd_relations表插入记录
|
||||
@Insert("insert into ${table} (gather_projects_ids) values (#{gather_projects_ids})")
|
||||
public void insertEddRelations(@Param("table") String table, @Param("gather_projects_ids") String gather_projects_ids);
|
||||
@Insert("insert into ${table} (gather_projects_ids) values (#{model.gather_projects_ids})")
|
||||
public void insertEddRelations(@Param("table") String table, @Param("model") EddRelations model);
|
||||
|
||||
/**
|
||||
* 下面是transferProjects程序的函数
|
||||
*/
|
||||
//获取edd_relations表中数据
|
||||
@Select("select * from ${table} where id>=#{start} limit #{size}")
|
||||
public List<EddRelations> getEddRelationList(@Param("table") String table, @Param("start") int start, @Param("size") int size);
|
||||
|
||||
//读取gather_projects表中的数据 根据id in来读取
|
||||
@Select("select * from ${table} where id=#{id}")
|
||||
public GatherProjectsModel getGatherProjectById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//读取synonymmings表
|
||||
@Select("select * from ${table} where synonymming_id=#{synonymming_id}")
|
||||
public List<Synonymmings> getSynonymmingList(@Param("table") String table, @Param("synonymming_id") int synonymming_id);
|
||||
|
||||
//读取synonym对象
|
||||
@Select("select * from ${table} where id=#{id}")
|
||||
public Synonyms getSynonymById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//更新edd_relations表的osp_id字段
|
||||
@Update("update ${table} set osp_id=#{osp_id} where id=#{id}")
|
||||
public void updateEddRelationOspId(@Param("table") String table, @Param("id") int id, @Param("osp_id") int osp_id);
|
||||
|
||||
//更新edd_relations表的gather_projects_ids字段
|
||||
@Update("update ${table} set gather_projects_ids=#{gather_projects_ids} where id=#{id}")
|
||||
public void updateEddRelationGatherProjectsIds(@Param("table") String table, @Param("id") int id, @Param("gather_projects_ids") String gather_projects_ids);
|
||||
|
||||
|
||||
//更新open_source_projects表中每条记录的synonyms字段
|
||||
@Update("update ${table} set synonyms=#{synonyms} where id=#{id}")
|
||||
public void updateSynonyms(@Param("table") String table, @Param("synonyms") String synonyms, @Param("id") int id);
|
||||
|
||||
|
||||
//读取open_source_projects表中的记录
|
||||
@Select("select * from ${table} where id>=#{startId} limit #{size}")
|
||||
public List<OpenSourceProject> getOpenSourceProjectList(@Param("table") String table, @Param("startId") int startId, @Param("size") int size);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
package com.ossean.databaseSource;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
import org.apache.ibatis.annotations.Update;
|
||||
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
|
||||
public interface GatherDao {
|
||||
|
||||
//向gather_tasks表插入转移任务
|
||||
@Insert("INSERT ignore INTO ${taskTableName} (BeginId,EndId,SourceTableName,TargetTableName) VALUES (${idsBegin},${idsEnd},#{sourceTableName},#{targetTableName})")
|
||||
public int insertLog(@Param("idsBegin") int idsBegin, @Param("idsEnd") int idsEnd, @Param("taskTableName") String taskTableName,
|
||||
@Param("sourceTableName") String sourceTableName, @Param("targetTableName") String targetTableName);
|
||||
|
||||
//查询未处理的task
|
||||
@Select("SELECT BeginId,EndId,BeginTime,EndTime " +
|
||||
"FROM ${migrationTask} " +
|
||||
"WHERE IsDone = 0 AND SourceTableName=#{sourceTableName} AND TargetTableName=#{targetTableName} ORDER BY BeginId LIMIT 0,1")
|
||||
public Map checkDone(@Param("migrationTask") String migrationTask, @Param("sourceTableName") String sourceTableName, @Param("targetTableName") String targetTableName);
|
||||
|
||||
//更新转移任务开始时间
|
||||
@Update("UPDATE ${migrationTask} SET BeginTime = NOW() WHERE BeginId = ${sourceIdBegin} AND SourceTableName=#{sourceTableName} AND TargetTableName=#{targetTableName}")
|
||||
public int updateTaskBeginTime(@Param("migrationTask") String migrationTask, @Param("sourceIdBegin") int sourceIdBegin, @Param("sourceTableName") String sourceTableName, @Param("targetTableName") String targetTableName);
|
||||
|
||||
//获取数据源中相应数据 两边都有等于号
|
||||
@Select("Select ${selectItems} from ${sourceTable} " +
|
||||
"where id >=${idBegin} and id <= ${idEnd} ${andWhere}")
|
||||
public List<GatherProjectsModel> moveDataGet(@Param("sourceTable") String sourceTableName,
|
||||
@Param("selectItems") String selectItems,
|
||||
@Param("idBegin") int idBegin, @Param("idEnd") int idEnd,
|
||||
@Param("andWhere") String andWhere);
|
||||
|
||||
//将获取到得数据源数据插入目标表中
|
||||
@Insert("INSERT INTO ${targetTable} (${targetFields}) VALUES (#{model.id},#{model.deal_name},#{model.name},#{model.description},#{model.last_update_time},#{model.registered_time},#{model.followers_num},#{model.url},#{model.urlMD5},#{model.pageMD5},#{model.language},#{model.tags},#{model.contributors_num},#{model.monitor},#{model.status},#{model.download_num},#{model.visit_num},#{model.category},#{model.crawled_time},#{model.history},#{model.source},#{model.license},#{model.platform},#{model.homepage})")
|
||||
public int moveDataInsert(@Param("targetTable") String targetTableName,
|
||||
@Param("targetFields") String targetFields,
|
||||
@Param("model") GatherProjectsModel model);
|
||||
|
||||
//对urlMD5相同的数据进行replace into操作
|
||||
@Update("replace into ${targetTable} (${targetFields}) values (#{model.deal_name},#{model.name},#{model.description},#{model.last_update_time},#{model.registered_time},#{model.followers_num},#{model.url},#{model.urlMD5},#{model.pageMD5},#{model.language},#{model.tags},#{model.contributors_num},#{model.monitor},#{model.status},#{model.download_num},#{model.visit_num},#{model.category},#{model.crawled_time},#{model.history},#{model.source},#{model.license},#{model.platform},#{model.homepage})")
|
||||
public int moveDataReplace(@Param("targetTable") String targetTableName,
|
||||
@Param("targetFields") String targetFields,
|
||||
@Param("model") GatherProjectsModel model);
|
||||
|
||||
//对urlMD5相同的数据进行update操作
|
||||
@Update("update ${targetTable} set deal_name=#{model.deal_name},name=#{model.name},description=#{model.description},"
|
||||
+ "last_update_time=#{model.last_update_time},registered_time=#{model.registered_time},"
|
||||
+ "followers_num=#{model.followers_num},pageMD5=#{model.pageMD5},language=#{model.language},"
|
||||
+ "tags=#{model.tags},contributors_num=#{model.contributors_num},monitor=#{model.monitor},"
|
||||
+ "status=#{model.status},download_num=#{model.download_num},visit_num=#{model.visit_num},"
|
||||
+ "category=#{model.category},crawled_time=#{model.crawled_time},history=#{model.history},"
|
||||
+ "source=#{model.source},license=#{model.license},platform=#{model.platform},homepage=#{model.homepage} where id=#{id}")
|
||||
public void moveDataUpdate(@Param("targetTable") String targetTableName, @Param("model") GatherProjectsModel model, @Param("id") int id);
|
||||
|
||||
|
||||
// //获取刚刚插入gather_projects表中数据的id 由于gather_projects表中的自增id主键删除
|
||||
// @Select("select id from ${table} where urlMD5=#{model.urlMD5} and name=#{model.name} order by id desc limit 1")
|
||||
// public int getAutoIncrementIdFromTable(@Param("table") String table, @Param("model") GatherProjectsModel model);
|
||||
|
||||
|
||||
//更新数据
|
||||
@Update("")
|
||||
public void updateData(@Param("id") int id, @Param("model") GatherProjectsModel model);
|
||||
|
||||
|
||||
//更新任务结束时间
|
||||
@Update("UPDATE ${migrationTask} SET IsDone = 1,EndTime = NOW(),flow=${moveNum} WHERE BeginId = ${endIdBegin} AND SourceTableName=#{sourceTableName} AND TargetTableName=#{targetTableName}")
|
||||
public int updateTaskEndTime(@Param("migrationTask") String migrationTask, @Param("endIdBegin") int endIdBegin, @Param("moveNum") int moveNum, @Param("sourceTableName") String sourceTableName, @Param("targetTableName") String targetTableName);
|
||||
|
||||
|
||||
//向tag表存储数据
|
||||
@Insert("insert ignore into ${table} (name) values (#{name})")
|
||||
public void insertTag(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
//根据tag的name找tag的id
|
||||
@Select("select id from ${table} where name=#{name}")
|
||||
public int selectTagIdByName(@Param("table") String table, @Param("name") String name);
|
||||
|
||||
//根据urlMD5码找到GatherProjectModel对象
|
||||
@Select("select * from ${table} where urlMD5=#{urlMD5}")
|
||||
public GatherProjectsModel selectGPMByUrlMD5(@Param("table") String table, @Param("urlMD5") String urlMD5);
|
||||
|
||||
|
||||
//根据id和limit取GatherProjectsModel列表
|
||||
//@Select("select * from ${table} where id>=#{start} and id<=#{maxId} order by id asc limit #{size}")
|
||||
//public List<GatherProjectsModel> selectGPMList(@Param("table") String table, @Param("start") int start, @Param("size") int size, @Param("maxId") int maxId);
|
||||
@Select("select * from ${table} where update_mark = 0 limit #{size}")
|
||||
public List<GatherProjectsModel> selectGPMList(@Param("table") String table, @Param("size") int size);
|
||||
|
||||
@Select("select * from ${table} where id>=#{start} and id<#{end} limit #{size}")
|
||||
public List<GatherProjectsModel> selectGPMListBySize(@Param("table") String table, @Param("start") int start, @Param("end") int end, @Param("size") int size);
|
||||
|
||||
//根据id取GatherProjectsModel对象
|
||||
@Select("select * from ${table} where id=#{id}")
|
||||
public GatherProjectsModel selectGPMById(@Param("table") String table, @Param("id") int id);
|
||||
|
||||
//查找相同homepage的项目 但不能是正在处理的项目本身
|
||||
@Select("select * from ${table} where (homepage=#{homepage} or homepage=#{homepage1} or homepage=#{homepage2} or homepage=#{homepage3} or homepage=#{homepage4}) and id!=#{id}")
|
||||
public List<GatherProjectsModel> selectGPMBySameHomePage(@Param("table") String table, @Param("homepage") String homepage, @Param("homepage1") String homepage1, @Param("homepage2") String homepage2, @Param("homepage3") String homepage3, @Param("homepage4") String homepage4, @Param("id") int id, @Param("source") String source);
|
||||
|
||||
//查找相同name的项目
|
||||
@Select("select * from ${table} where name=#{name} and id!=#{id}")
|
||||
public List<GatherProjectsModel> selectGPMBySameName(@Param("table") String table, @Param("name") String name, @Param("id") int id);
|
||||
|
||||
|
||||
//取出所有update_mark为2的更新数据
|
||||
@Select("select * from ${table} where update_mark=2 limit #{size}")
|
||||
public List<GatherProjectsModel> selectUpdateProjects(@Param("table") String table,@Param("size") int size);
|
||||
|
||||
//更新update_mark字段
|
||||
@Select("update ${table} set update_mark=#{update_mark} where id=#{id}")
|
||||
public void updateUpdateMark(@Param("table") String table, @Param("update_mark") int update_mark, @Param("id") int id);
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.ossean.dao;
|
||||
package com.ossean.databaseSource;
|
||||
|
||||
import org.apache.ibatis.annotations.Insert;
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package com.ossean.databaseSource;
|
||||
|
||||
import org.apache.ibatis.annotations.Param;
|
||||
import org.apache.ibatis.annotations.Select;
|
||||
|
||||
import com.ossean.model.OpenhubProject;
|
||||
import com.ossean.model.SourceForgeProject;
|
||||
|
||||
|
||||
public interface PartProjectDao {
|
||||
|
||||
@Select("select name,description,code_repository from openhub_project where url = #{url} group by url_md5 order by extracted_time desc")
|
||||
public OpenhubProject getOpenHubPrjByUrl( @Param("url") String url);
|
||||
|
||||
@Select("select name,description,download_num,favor_num from sourceforge_project where url = #{url} group by url_md5 order by extracted_time desc")
|
||||
public SourceForgeProject getSourceForgePrjByUrl( @Param("url") String url);
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package com.ossean.dao;
|
||||
package com.ossean.databaseSource;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
|
|
@ -21,13 +21,6 @@ public class GatherProjectsModel {
|
|||
private int relative_memos_num;
|
||||
private String created_time;
|
||||
private String category;
|
||||
private int update_mark;
|
||||
public int getUpdate_mark() {
|
||||
return update_mark;
|
||||
}
|
||||
public void setUpdate_mark(int update_mark) {
|
||||
this.update_mark = update_mark;
|
||||
}
|
||||
public int getId() {
|
||||
return id;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,26 +1,25 @@
|
|||
package com.ossean.model;
|
||||
|
||||
public class OpenhubProject {
|
||||
private String description;
|
||||
private String name;
|
||||
private int rate_num;
|
||||
private int user_num;
|
||||
private String codeRepository;
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
public void setDescription(String description) {
|
||||
this.description = description;
|
||||
}
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
public int getRate_num() {
|
||||
return rate_num;
|
||||
public String getCodeRepository() {
|
||||
return codeRepository;
|
||||
}
|
||||
public void setRate_num(int rate_num) {
|
||||
this.rate_num = rate_num;
|
||||
public void setCodeRepository(String codeRepository) {
|
||||
this.codeRepository = codeRepository;
|
||||
}
|
||||
public int getUser_num() {
|
||||
return user_num;
|
||||
}
|
||||
public void setUser_num(int user_num) {
|
||||
this.user_num = user_num;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
package com.ossean.model;
|
||||
|
||||
public class OschinaProject {
|
||||
private String name;
|
||||
private int star_num;
|
||||
private int user_num;
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
public int getStar_num() {
|
||||
return star_num;
|
||||
}
|
||||
public void setStar_num(int star_num) {
|
||||
this.star_num = star_num;
|
||||
}
|
||||
public int getUser_num() {
|
||||
return user_num;
|
||||
}
|
||||
public void setUser_num(int user_num) {
|
||||
this.user_num = user_num;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -1,10 +1,16 @@
|
|||
package com.ossean.model;
|
||||
|
||||
public class SourceForgeProject {
|
||||
private String description;
|
||||
private String name;
|
||||
private int download_num;
|
||||
private int stars;
|
||||
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
public void setDescription(String description) {
|
||||
this.description = description;
|
||||
}
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +1,13 @@
|
|||
package com.ossean.model;
|
||||
|
||||
public class Synonymmings{
|
||||
public class Synonymmings {
|
||||
|
||||
private int id;
|
||||
private int synonym_id;
|
||||
private int synonymming_id;
|
||||
private int flag;
|
||||
public int getId() {
|
||||
return id;
|
||||
}
|
||||
public int getFlag() {
|
||||
return flag;
|
||||
}
|
||||
public void setFlag(int flag) {
|
||||
this.flag = flag;
|
||||
}
|
||||
public void setId(int id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
package com.ossean.model;
|
||||
|
||||
public class synonymmings_new {
|
||||
|
||||
private int id;
|
||||
private int prjId;
|
||||
private String synonyms;
|
||||
private int flag;
|
||||
public int getId() {
|
||||
return id;
|
||||
}
|
||||
public void setId(int id) {
|
||||
this.id = id;
|
||||
}
|
||||
public int getPrjId() {
|
||||
return prjId;
|
||||
}
|
||||
public void setPrjId(int prjId) {
|
||||
this.prjId = prjId;
|
||||
}
|
||||
public String getSynonyms() {
|
||||
return synonyms;
|
||||
}
|
||||
public void setSynonyms(String synonyms) {
|
||||
this.synonyms = synonyms;
|
||||
}
|
||||
public int getFlag() {
|
||||
return flag;
|
||||
}
|
||||
public void setFlag(int flag) {
|
||||
this.flag = flag;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
package com.ossean.util;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.ossean.databaseDest.DBDest;
|
||||
import com.ossean.databaseSource.DBSource;
|
||||
import com.ossean.databaseSource.GatherDao;
|
||||
import com.ossean.databaseSource.PKControlProjectsDao;
|
||||
import com.ossean.databaseSource.UpdateControlProjectsDao;
|
||||
import com.ossean.model.Configure;
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.model.PKControlProjects;
|
||||
import com.ossean.model.Taggings;
|
||||
import com.ossean.model.UpdateControlProjects;
|
||||
|
||||
@Component("gatherProjectsUtil")
|
||||
public class GatherProjectsUtil {
|
||||
Logger logger = Logger.getLogger(this.getClass());
|
||||
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
@Resource
|
||||
private DBDest dbDest;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
@Resource
|
||||
private PKControlProjectsDao pkControlProjectsDao;
|
||||
@Resource
|
||||
private UpdateControlProjectsDao updateControlProjectsDao;
|
||||
|
||||
private String gatherProjectsTableName = "gather_projects";
|
||||
|
||||
private String taggingsTableName = "edd_taggings";
|
||||
private String tagsTableName = "edd_tags";
|
||||
|
||||
|
||||
private String pkControlProjectsTableName = "pk_control_projects";
|
||||
private String updateControlProjectsTableName = "update_control_projects";
|
||||
|
||||
|
||||
//批量处理转移数据 需要处理为原子操作
|
||||
@Transactional(propagation=Propagation.REQUIRED)
|
||||
public void handleBatchData(Map checkDoneMap, Configure conf){
|
||||
logger.info("BeginId#"+conf.getSourceTableName()+":"+ checkDoneMap.get("BeginId"));
|
||||
int sourceIdBegin = Integer.parseInt(checkDoneMap.get("BeginId").toString());
|
||||
int sourceIdEnd = Integer.parseInt(checkDoneMap.get("EndId").toString());
|
||||
|
||||
if (checkDoneMap.get("EndTime") == null) {
|
||||
//表示任务没有完成
|
||||
int maxId = dbSource.getMaxId(conf.getSourceTableName());
|
||||
|
||||
// 防止转移超过当前最大值的Id数据
|
||||
if (sourceIdBegin >= 0 && sourceIdEnd > 0 && maxId >= sourceIdEnd) {
|
||||
// 更新执行开始时间
|
||||
gatherDao.updateTaskBeginTime(conf.getMigrationTask(),sourceIdBegin,conf.getSourceTableName(),conf.getTargetTableName());
|
||||
logger.info("Transaction...@"+ new Date());
|
||||
// 插入Id段数据，忽略重复值
|
||||
int num = 0;
|
||||
try {
|
||||
String[] sourceFields=conf.getSourceFields().split(",");
|
||||
String[] targetFields = conf.getTargetFields().split(",");
|
||||
String selectItems = "";
|
||||
for(int i = 0; i < sourceFields.length; i++){
|
||||
String str_source = sourceFields[i];
|
||||
String str_target = targetFields[i];
|
||||
selectItems += str_source + " as " + str_target + ",";
|
||||
}
|
||||
selectItems = selectItems.substring(0, selectItems.length() - 1) + " ";
|
||||
List<GatherProjectsModel> dataGet = gatherDao.moveDataGet(
|
||||
conf.getSourceTableName(),selectItems,
|
||||
sourceIdBegin,sourceIdEnd,conf.getAndWhere());
|
||||
|
||||
num = dataGet.size();
|
||||
for(int i = 0; i < dataGet.size(); i++){
|
||||
GatherProjectsModel model = dataGet.get(i);
|
||||
String urlMD5 = model.getUrl_md5();//通过urlMD5判断是不是已经存在过该项目 是否更新
|
||||
GatherProjectsModel sameProject = gatherDao.selectGPMByUrlMD5(conf.getTargetTableName(), urlMD5);
|
||||
if(sameProject == null){
|
||||
//不存在 插入
|
||||
PKControlProjects pkControlModel = pkControlProjectsDao.selectItemByUrlMD5(pkControlProjectsTableName, urlMD5);//查看有没有固定的id
|
||||
if(pkControlModel != null)
|
||||
model.setId(pkControlModel.getId());
|
||||
else{
|
||||
//在pk_control_projects表中生成当前项目对应的id
|
||||
pkControlProjectsDao.insertOneItem(pkControlProjectsTableName, urlMD5);
|
||||
//查看刚刚插入信息的id
|
||||
PKControlProjects controlItem = pkControlProjectsDao.selectItemByUrlMD5(pkControlProjectsTableName, urlMD5);
|
||||
//用id构造model对应的固定不变的id
|
||||
model.setId(controlItem.getId());
|
||||
}
|
||||
handleInsertGatherProjects(model, conf);
|
||||
}else{
|
||||
//update gather_projects表中对应的记录，在维持待更新表
|
||||
handleUpdateGatherProjects(sameProject.getId(),model);
|
||||
}
|
||||
|
||||
//将tag和项目的关系存入表item_tag_relation 并分离tag
|
||||
String tags = model.getTags();
|
||||
if(tags == null){
|
||||
//表示该项目没有标签
|
||||
continue;
|
||||
}
|
||||
String[] tagArray = tags.split(",");
|
||||
for(String tag:tagArray){
|
||||
tag = tag.substring(1, tag.length() - 1).toLowerCase();//去掉两边的尖括号 并转换为小写
|
||||
gatherDao.insertTag(tagsTableName, tag);//ignore方式插入该项目的标签
|
||||
int tag_id = gatherDao.selectTagIdByName(tagsTableName, tag);
|
||||
Taggings taggings = new Taggings();
|
||||
taggings.setTag_id(tag_id);
|
||||
taggings.setUrlMD5(urlMD5);
|
||||
taggings.setTaggable_type("OpenSourceProject");
|
||||
taggings.setContext("tags");
|
||||
taggings.setCreated_at(DateHandler.getNow());
|
||||
taggings.setDisagree_num(50);
|
||||
//将Taggings对象存入数据库中
|
||||
try {
|
||||
dbSource.insertTaggings(taggingsTableName, taggings);
|
||||
} catch (Exception e) {
|
||||
//在插入记录之前 gather_projects表中的记录已经被删除掉了
|
||||
System.out.println(e.getMessage());
|
||||
System.exit(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
// 数据迁移过程可能发生异常情况
|
||||
logger.error(ex.toString());
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
// 更新执行结束时间
|
||||
gatherDao.updateTaskEndTime(conf.getMigrationTask(),sourceIdBegin, num,conf.getSourceTableName(),conf.getTargetTableName());
|
||||
logger.info("EndTime:" + new Date());
|
||||
|
||||
// 更新游标到本次 EndId+1;
|
||||
dbSource.updatePointer(conf.getPointerTableName(), conf.getSourceTableName(), conf.getTargetTableName()
|
||||
, sourceIdEnd + 1);//sourceIdBegin + idsIncrement
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//处理URL不存在的项目 插入gather_projects表
|
||||
public void handleInsertGatherProjects(GatherProjectsModel model, Configure conf){
|
||||
gatherDao.moveDataInsert(conf.getTargetTableName(),conf.getTargetFields(), model);
|
||||
// int taggable_id = gatherDao.getAutoIncrementIdFromTable(conf.getTargetTableName(), model);
|
||||
}
|
||||
|
||||
//处理URL相同的项目更新 id表示更新的项目固定id
|
||||
public void handleUpdateGatherProjects(int id, GatherProjectsModel model_new){
|
||||
//删除gather_projects表中原记录
|
||||
// dbSource.deleteItemById(gatherProjectsTableName, id);
|
||||
// List<EddRelations> eddRelationList = dbSource.getEddRelationsByGatherProjectsId(eddRelationTableName, id);
|
||||
// for(EddRelations relation:eddRelationList){
|
||||
// if(relation.getOsp_id() != null){
|
||||
// //删除osp_id对应的open_source_projects表中的记录
|
||||
// dbDest.deleteOpenSourceProjectsItem(openSourceProjectsTableName, relation.getOsp_id());
|
||||
// }
|
||||
// //删除edd_relations表中的记录
|
||||
// dbSource.deleteEddRelationsItem(eddRelationTableName, relation.getId());
|
||||
// }
|
||||
// //如果有剩余的关联 插入剩余的关联
|
||||
// String relation_new = ",";
|
||||
// Map<String, Integer> relationMap = getRelationMap(eddRelationList);
|
||||
// Set<String> keys = relationMap.keySet();
|
||||
// Iterator<String> it = keys.iterator();
|
||||
// while(it.hasNext()){
|
||||
// String key = it.next();
|
||||
// if(Integer.parseInt(key) != id)
|
||||
// relation_new = relation_new + key + ",";//如果是已经删除的id 就不添加到新的relation字符串中
|
||||
// }
|
||||
// if(relation_new.length() > 1){
|
||||
// //表示有剩余的关联需要插入edd_relations表中
|
||||
// EddRelations r = new EddRelations();
|
||||
// r.setGather_projects_ids(relation_new);
|
||||
// dbSource.insertEddRelations(eddRelationTableName, r);
|
||||
// }
|
||||
//
|
||||
// //在gather_projects表中插入新记录
|
||||
// gatherDao.moveDataInsert(conf.getTargetTableName(), conf.getTargetFields() , model_new);
|
||||
// int taggable_id = gatherDao.getAutoIncrementIdFromTable(conf.getTargetTableName(), model_new);
|
||||
// return taggable_id;
|
||||
|
||||
gatherDao.moveDataUpdate(gatherProjectsTableName, model_new, id);//更新数据gather_projects表
|
||||
//更新维持更新表 update_control_projects
|
||||
//首先判断表前面有没有当前项目还没有再次更新的
|
||||
UpdateControlProjects sameUpdate = updateControlProjectsDao.findSameUpdate(updateControlProjectsTableName, id);
|
||||
if(sameUpdate == null){
|
||||
//表示没有相同的update 则直接插入
|
||||
updateControlProjectsDao.insertOneItem(updateControlProjectsTableName, id);
|
||||
}else{
|
||||
//表示有相同的更新请求 更新时间
|
||||
updateControlProjectsDao.updateItem(updateControlProjectsTableName, sameUpdate);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
package com.ossean.util;
|
||||
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
|
||||
public class MD5Handler {
|
||||
|
||||
// 全局数组
|
||||
private final static String[] strDigits = { "0", "1", "2", "3", "4", "5",
|
||||
"6", "7", "8", "9", "a", "b", "c", "d", "e", "f" };
|
||||
|
||||
public static String getMD5(String strObj){
|
||||
String resultString = null;
|
||||
try {
|
||||
resultString = new String(strObj);
|
||||
MessageDigest md = MessageDigest.getInstance("MD5");
|
||||
// md.digest() 该函数返回值为存放哈希值结果的byte数组
|
||||
resultString = byteToString(md.digest(strObj.getBytes()));
|
||||
} catch (NoSuchAlgorithmException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
return resultString;
|
||||
}
|
||||
|
||||
|
||||
private static String byteToString(byte[] bByte) {
|
||||
StringBuffer sBuffer = new StringBuffer();
|
||||
for (int i = 0; i < bByte.length; i++) {
|
||||
sBuffer.append(byteToArrayString(bByte[i]));
|
||||
}
|
||||
return sBuffer.toString();
|
||||
}
|
||||
|
||||
// 返回形式为数字跟字符串
|
||||
private static String byteToArrayString(byte bByte) {
|
||||
int iRet = bByte;
|
||||
// System.out.println("iRet="+iRet);
|
||||
if (iRet < 0) {
|
||||
iRet += 256;
|
||||
}
|
||||
int iD1 = iRet / 16;
|
||||
int iD2 = iRet % 16;
|
||||
return strDigits[iD1] + strDigits[iD2];
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,414 +0,0 @@
|
|||
package com.ossean.util;
|
||||
|
||||
import info.debatty.java.stringsimilarity.NGram;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.io.StringReader;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.wltea.analyzer.core.IKSegmenter;
|
||||
import org.wltea.analyzer.core.Lexeme;
|
||||
|
||||
import com.ossean.MergeProjects;
|
||||
import com.ossean.TableName;
|
||||
import com.ossean.dao.DBDest;
|
||||
import com.ossean.dao.DBSource;
|
||||
import com.ossean.dao.GatherDao;
|
||||
import com.ossean.dao2.PlatformProjectDao;
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.model.Synonymmings;
|
||||
import com.ossean.model.Synonyms;
|
||||
import com.ossean.model.Taggings;
|
||||
import com.ossean.model.Tags;
|
||||
|
||||
@Component("mergeProjectNew2")
|
||||
public class MergeProjectNew2 {
|
||||
Logger logger = Logger.getLogger(this.getClass());
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
@Resource
|
||||
private PlatformProjectDao platformProjectDao;
|
||||
@Resource
|
||||
private DBDest dbDest;
|
||||
private String synonymsTableName = TableName.synonymsTableName;
|
||||
private String gatherProjectsTableName = TableName.gatherProjectsTableName;
|
||||
private String eddRelationTableName = TableName.eddRelationTableName;
|
||||
|
||||
private static Set<Integer> passSet;
|
||||
private static Set<Integer> relationSet;
|
||||
|
||||
public List<String> getSynonyms(GatherProjectsModel model){
|
||||
List<String> synonymsList = new ArrayList<String>();//最终返回的同义词列表
|
||||
synonymsList = dbSource.getSynonymByPrjId(synonymsTableName, model.getId());
|
||||
return synonymsList;
|
||||
}
|
||||
//处理新项目
|
||||
|
||||
public Set<Integer> handleNewProject(GatherProjectsModel model){
|
||||
passSet = new HashSet<Integer>();//已经处理的项目，避免重复处理陷入死锁
|
||||
relationSet = new HashSet<Integer>();//所有重复的项目集合
|
||||
relationSet = findDupProject(model,relationSet,passSet);
|
||||
OperateTable(relationSet, model.getId());//表操作
|
||||
return relationSet;
|
||||
}
|
||||
|
||||
public Set<Integer> findDupProject(GatherProjectsModel model,Set<Integer> relationSet,Set<Integer> passSet){
|
||||
|
||||
String prjName = model.getName();
|
||||
int prjId = model.getId();
|
||||
String prjHomepage = model.getHomepage();
|
||||
|
||||
//项目重复匹配列表
|
||||
List<GatherProjectsModel> matchedList = new ArrayList<GatherProjectsModel>();
|
||||
List<GatherProjectsModel> sameHomepageList = new ArrayList<GatherProjectsModel>();
|
||||
|
||||
//apache社区没有homepage,用url代替
|
||||
if(prjHomepage == null || prjHomepage.equals(""))
|
||||
if(model.getSource().equals("apache"))
|
||||
prjHomepage = model.getUrl();
|
||||
//homepage相同认为一定是相同的项目
|
||||
if(prjHomepage != null && !prjHomepage.equals("")){
|
||||
prjHomepage = deleteHttpPre(prjHomepage);//去掉“http&https”前缀及“/”后缀
|
||||
if(prjHomepage.length() > 0){
|
||||
//为homepage添加http或https头
|
||||
String homepage1 = "http://" + prjHomepage;
|
||||
String homepage2 = "https://" + prjHomepage;
|
||||
String homepage3 = "http://" + prjHomepage + "/";
|
||||
String homepage4 = "https://" + prjHomepage + "/";
|
||||
sameHomepageList = gatherDao.selectGPMBySameHomePage(gatherProjectsTableName, prjHomepage, homepage1, homepage2, homepage3, homepage4, model.getId(),1);
|
||||
for(GatherProjectsModel m : sameHomepageList){
|
||||
//logger.info("project " + model.getId() + " and " + m.getId() + " match by homepage");
|
||||
if(!passSet.contains(m.getId()))
|
||||
matchedList.add(m);
|
||||
}
|
||||
}
|
||||
}
|
||||
//处理同名项目
|
||||
List<GatherProjectsModel> sameNamePrjList = gatherDao.selectGPMBySameName(gatherProjectsTableName, prjName, prjId,1);
|
||||
List<GatherProjectsModel> afterNamePrjList = new ArrayList<GatherProjectsModel>();
|
||||
for(int i = 0;i<sameNamePrjList.size();i++){
|
||||
//logger.info("project " + model.getId() + " and " + sameNamePrjList.get(i).getId() + " match by name");
|
||||
if(!passSet.contains(sameNamePrjList.get(i).getId()))//未处理的
|
||||
afterNamePrjList.add(sameNamePrjList.get(i));
|
||||
}
|
||||
afterNamePrjList = returnDupModelList(model,afterNamePrjList,true);
|
||||
//处理别名相同的项目
|
||||
List<String> synonymList = getSynonyms(model);//获取项目的同义词
|
||||
List<GatherProjectsModel> synonymsPrjList = getRelatedGatherProjects(model,synonymList);
|
||||
List<GatherProjectsModel> afterSynonymsPrjList = new ArrayList<GatherProjectsModel>();
|
||||
for(int j = 0;j < synonymsPrjList.size();j++){
|
||||
//logger.info("project " + model.getId() + " and " + synonymsPrjList.get(j).getId() + " match by synonyms");
|
||||
if(!passSet.contains(synonymsPrjList.get(j).getId()) && !afterNamePrjList.contains(model))//未处理的,也不同名，否则会重复计算
|
||||
afterSynonymsPrjList.add(synonymsPrjList.get(j));
|
||||
}
|
||||
afterSynonymsPrjList = returnDupModelList(model,afterSynonymsPrjList,false);
|
||||
for(GatherProjectsModel m : afterSynonymsPrjList){
|
||||
//logger.info("project " + model.getId() + " and " + m.getId() + " match by synonyms/name and verify");
|
||||
}
|
||||
afterSynonymsPrjList = mergeTwoList(afterSynonymsPrjList,afterNamePrjList);
|
||||
matchedList = mergeTwoList4Source(matchedList,afterSynonymsPrjList);//同一社区的项目相同homepage比名字相同和别名相同优先级更高
|
||||
relationSet.add(model.getId());
|
||||
passSet.add(model.getId());
|
||||
if(matchedList.size()==0)
|
||||
return relationSet;
|
||||
|
||||
//递归调用有重复关系的项目
|
||||
for(int i = 0; i < matchedList.size(); i++){
|
||||
GatherProjectsModel dupModel = matchedList.get(i);
|
||||
if(!passSet.contains(dupModel.getId())){
|
||||
findDupProject(dupModel,relationSet,passSet);
|
||||
}
|
||||
}
|
||||
return relationSet;
|
||||
}
|
||||
//分别对别名和关系进行插表操作：所有别名汇聚到正在处理的项目上(handlenewproject)。
|
||||
//对于重复关系的项目标记update_mark=2，之后不再重复操作
|
||||
@Transactional(propagation=Propagation.REQUIRED)
|
||||
public void OperateTable(Set<Integer> relationSet,int prjId){
|
||||
List<String> dupSynonymsList = new ArrayList<String>();
|
||||
String relationStr = ",";
|
||||
for (int id:relationSet) {//该集合包括处理项目本身
|
||||
relationStr = relationStr+id+",";
|
||||
dupSynonymsList.addAll(dbSource.getSynonymByPrjId(synonymsTableName, id));//获取flag=1的同义词（确定的）
|
||||
gatherDao.updateMark(gatherProjectsTableName, 2, id);
|
||||
}
|
||||
dupSynonymsList = removeDupStr(dupSynonymsList);
|
||||
for(String synonyms:dupSynonymsList){
|
||||
dbSource.insertSynonymsNew(synonymsTableName,prjId,synonyms,1);
|
||||
}
|
||||
//更新标志位，方便到时候取得重复项目的所有别名
|
||||
//dbSource.updateSynonyms(synonymsTableName,prjId,1);
|
||||
dbSource.insertEddRelations(eddRelationTableName, relationStr);
|
||||
}
|
||||
|
||||
//找到当前项目同义词关联的项目 要求项目不相同 因为同一个项目可能包含多个同义词
|
||||
public List<GatherProjectsModel> getRelatedGatherProjects(GatherProjectsModel model, List<String> synonymList){
|
||||
|
||||
List<GatherProjectsModel> result = new ArrayList<GatherProjectsModel>();
|
||||
Map<Integer,GatherProjectsModel> map = new HashMap<Integer,GatherProjectsModel>();
|
||||
GatherProjectsModel tmp;
|
||||
for(String name:synonymList){
|
||||
List<Integer> prjList= dbSource.getSynonymBySynonyms(synonymsTableName, name);
|
||||
for(int prjId:prjList){
|
||||
if(prjId==model.getId())
|
||||
continue;
|
||||
if(map.containsKey(prjId))
|
||||
continue;
|
||||
tmp = gatherDao.selectGPMById(gatherProjectsTableName, prjId);
|
||||
if(tmp.getUpdate_mark()==2)
|
||||
continue;
|
||||
map.put(prjId, tmp);
|
||||
}
|
||||
}
|
||||
for (GatherProjectsModel value : map.values()) {
|
||||
result.add(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public List<GatherProjectsModel> returnDupModelList(GatherProjectsModel model,List<GatherProjectsModel> list,boolean samename){
|
||||
List<GatherProjectsModel> returnList = new ArrayList<GatherProjectsModel>();
|
||||
for(GatherProjectsModel dupModel:list){
|
||||
if(dupModel.getSource()!=model.getSource()){
|
||||
if(isTheSame(model,dupModel,samename))
|
||||
returnList.add(dupModel);
|
||||
}
|
||||
}
|
||||
//如果重复列表有来自同一社区的，则再进行比较相似度
|
||||
List<GatherProjectsModel> afterReturnList = new ArrayList<GatherProjectsModel>();
|
||||
Map<String,List<GatherProjectsModel>> map = new HashMap<String, List<GatherProjectsModel>>();
|
||||
GatherProjectsModel uniqueModel = new GatherProjectsModel();
|
||||
for(GatherProjectsModel model1:returnList){
|
||||
if(map.get(model1.getSource())==null){
|
||||
map.put(model1.getSource(),new ArrayList<GatherProjectsModel>());
|
||||
map.get(model1.getSource()).add(model1);
|
||||
}
|
||||
else
|
||||
map.get(model1.getSource()).add(model1);
|
||||
}
|
||||
for(String uniqueSource:map.keySet()){
|
||||
uniqueModel = getUniqueSourceModel(map.get(uniqueSource));
|
||||
afterReturnList.add(uniqueModel);
|
||||
}
|
||||
return afterReturnList;
|
||||
|
||||
}
|
||||
|
||||
public GatherProjectsModel getUniqueSourceModel(List<GatherProjectsModel> list){
|
||||
GatherProjectsModel returnModel = new GatherProjectsModel();
|
||||
String source = list.get(0).getSource();
|
||||
if(list.size()==1)
|
||||
returnModel = list.get(0);
|
||||
else{
|
||||
GatherProjectsModel tmp = list.get(0);
|
||||
|
||||
for(GatherProjectsModel model1:list){
|
||||
if(source.equals("apache")||source.equals("freecode"))
|
||||
returnModel = list.get(0);
|
||||
// if(source.equals("oschina")){
|
||||
// int a = platformProjectDao.getOschinaPrjByUrl(model1.getUrl()).getStar_num();
|
||||
// int b = platformProjectDao.getOschinaPrjByUrl(tmp.getUrl()).getStar_num();
|
||||
// if(a > b){
|
||||
// tmp = model1;
|
||||
// }
|
||||
// }
|
||||
if(source.equals("openhub")){
|
||||
int a = platformProjectDao.getOpenHubPrjByUrl(model1.getUrl()).getUser_num();
|
||||
int b = platformProjectDao.getOpenHubPrjByUrl(tmp.getUrl()).getUser_num();
|
||||
if(a > b){
|
||||
tmp = model1;
|
||||
}
|
||||
}
|
||||
if(source.equals("sourceforge")){
|
||||
int a = platformProjectDao.getSourceForgePrjByUrl(model1.getUrl()).getDownload_num();
|
||||
int b = platformProjectDao.getSourceForgePrjByUrl(tmp.getUrl()).getDownload_num();
|
||||
if(a > b){
|
||||
tmp = model1;
|
||||
}
|
||||
}
|
||||
}
|
||||
returnModel = tmp;
|
||||
}
|
||||
return returnModel;
|
||||
}
|
||||
|
||||
|
||||
public boolean isTheSame(GatherProjectsModel model1, GatherProjectsModel model2,boolean sameName){
|
||||
double similarity;
|
||||
int similarTagNum;
|
||||
boolean isEqual=false;
|
||||
//oschina的tag和description都为中文
|
||||
if(model1.getSource().equals("oschina")||model2.getSource().equals("oschina")){
|
||||
if(sameName){
|
||||
if(model1.getSource().equals("oschina"))
|
||||
isEqual = StringHandler.compareLanguage(model1.getLanguage(),model2.getLanguage());
|
||||
if(model2.getSource().equals("oschina"))
|
||||
isEqual = StringHandler.compareLanguage(model2.getLanguage(),model1.getLanguage());
|
||||
if (isEqual)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
//获取两个对象的tags属性
|
||||
String tags1 = model1.getTags();
|
||||
String tags2 = model2.getTags();
|
||||
similarTagNum = calSimilarityByTag(model1, model2);
|
||||
//System.out.println("calSimilarityByTag "+model1.getId()+" and "+model2.getId()+" : "+String.valueOf(similarTagNum));
|
||||
if(sameName){//相同名字的项目
|
||||
if(similarTagNum >=1)
|
||||
return true;
|
||||
else{
|
||||
similarity = calSimilarity(model1, model2);
|
||||
//System.out.println("same name to calSimilarityByDescrption between "+model1.getId()+" and "+model2.getId()+" : "+similarity);
|
||||
if(similarity>0.2)
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else{//相同别名的项目
|
||||
if(similarTagNum >1)
|
||||
return true;
|
||||
else{
|
||||
similarity = calSimilarity(model1, model2);
|
||||
System.out.println("same synonyms to calSimilarityByDescrption between "+model1.getId()+" and "+model2.getId()+" : "+similarity);
|
||||
if(similarity>=0.4)
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public double calSimilarity(GatherProjectsModel model1, GatherProjectsModel model2){
|
||||
NGram ngram = new NGram(4);
|
||||
return (1-ngram.distance(model1.getDescription(),model2.getDescription()));
|
||||
}
|
||||
//计算都有标签的项目之间的相似度
|
||||
public int calSimilarityByTag(GatherProjectsModel model1, GatherProjectsModel model2){
|
||||
if(model1.getTags()==null || model1.getTags().equals("")||model2.getTags()==null ||model2.getTags().equals(""))
|
||||
return 0;
|
||||
String[] tagArray_model1 = model1.getTags().split(",");
|
||||
String[] tagArray_model2 = model2.getTags().split(",");
|
||||
if (model1.getSource().equals("sourceforge"))
|
||||
tagArray_model1 = StringHandler.splitTagsByBracket(model1.getTags());
|
||||
if (model2.getSource().equals("sourceforge"))
|
||||
tagArray_model2 = StringHandler.splitTagsByBracket(model2.getTags());
|
||||
List<String> tags_model1 = changeArrayToList(tagArray_model1);
|
||||
List<String> tags_model2 = changeArrayToList(tagArray_model2);
|
||||
int sameNum = getSameTagsNum(tags_model1,tags_model2);
|
||||
return sameNum;
|
||||
}
|
||||
|
||||
public List<String> changeArrayToList(String[] strs){
|
||||
List<String> result = new ArrayList<String>();
|
||||
for(String str:strs){
|
||||
str = RegexHandler.extractEngDecimalAndChinese(str);//对标签进行处理
|
||||
result.add(str);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public String deleteHttpPre(String homepage){
|
||||
if(homepage != null && !"".equals(homepage)){
|
||||
homepage = homepage.trim();
|
||||
int index_http = homepage.indexOf("http://");
|
||||
int index_https = homepage.indexOf("https://");
|
||||
if(index_http == 0){
|
||||
homepage = homepage.substring(index_http + 7); //去除http://
|
||||
}else if(index_https == 0){
|
||||
homepage = homepage.substring(index_https + 8); //去除https://
|
||||
}else{
|
||||
//表示不是以http或https开头
|
||||
}
|
||||
|
||||
if((homepage.lastIndexOf("/") == homepage.length()-1) && (homepage.length() != 0)){
|
||||
homepage = homepage.substring(0, homepage.length() - 1);//如果homepage的最后一个字符是/ 需要去除掉
|
||||
}
|
||||
}
|
||||
return homepage;
|
||||
}
|
||||
|
||||
public List<String> removeDupStr(List<String> array){
|
||||
List<String> list = new ArrayList<String>();
|
||||
Set<String> set = new HashSet<String>();
|
||||
for(String s : array){
|
||||
if(set.add(s.toLowerCase())){
|
||||
list.add(s);
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
//将两个list<GatherProjectsModel>进行合并
|
||||
public List<GatherProjectsModel> mergeTwoList(List<GatherProjectsModel> list1, List<GatherProjectsModel> list2){
|
||||
List<GatherProjectsModel> result = new ArrayList<GatherProjectsModel>();
|
||||
Set set = new HashSet<Integer>();
|
||||
for(GatherProjectsModel model:list1){
|
||||
if(set.add(model.getId()))
|
||||
result.add(model);
|
||||
}
|
||||
for(GatherProjectsModel model:list2){
|
||||
if(set.add(model.getId()))
|
||||
result.add(model);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public List<GatherProjectsModel> mergeTwoList4Source(List<GatherProjectsModel> list1, List<GatherProjectsModel> list2){
|
||||
List<GatherProjectsModel> result = new ArrayList<GatherProjectsModel>();
|
||||
Set set = new HashSet<Integer>();
|
||||
Set sourceSet = new HashSet<String>();
|
||||
for(GatherProjectsModel model:list1){
|
||||
sourceSet.add(model.getSource());
|
||||
if(set.add(model.getId()))
|
||||
result.add(model);
|
||||
}
|
||||
for(GatherProjectsModel model:list2){
|
||||
if(set.add(model.getId())&& sourceSet.add(model.getSource()))
|
||||
result.add(model);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public int getSameTagsNum(List<String> list1, List<String> list2){
|
||||
int count = 0;
|
||||
for (String str:list1){
|
||||
if(isExist(str,list2))
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
|
||||
}
|
||||
|
||||
public boolean isExist(String input,List<String> list){
|
||||
boolean b=false;
|
||||
for(String str:list){
|
||||
if(str.toLowerCase().equals(input.toLowerCase()))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,12 +1,9 @@
|
|||
package com.ossean.util;
|
||||
|
||||
import java.security.acl.LastOwnerException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class StringHandler {
|
||||
|
||||
|
|
@ -52,39 +49,6 @@ public class StringHandler {
|
|||
return resultStrArr;
|
||||
}
|
||||
|
||||
public static boolean compareLanguage(String str1,String str2){
|
||||
|
||||
//str1是来自oschina的项目语言信息
|
||||
/*
|
||||
*openhub: C++|:|55%|,|JavaScript|:|21%|,or null
|
||||
*sourceforge:JavaScript, Groovy, Java or null
|
||||
*oschina:<PHP>,<JavaScript> or null
|
||||
*apache:一般只有一个
|
||||
*freecode:<jdbc>,<Java>,<Database development>
|
||||
*/
|
||||
//如果oschina没有language，则返回false
|
||||
if (str1==null||str1.equals("")||str2==null||str2.equals(""))
|
||||
return false;
|
||||
String []arr = str1.split(",");
|
||||
Pattern p = Pattern.compile("<(.*?)>");
|
||||
for(int i=0;i<arr.length;i++){
|
||||
Matcher m = p.matcher(arr[i]);
|
||||
if(m.find()) {
|
||||
String tmp = m.group(1)+",";
|
||||
String tmp2 = m.group(1)+"|";
|
||||
String tmp3 = "<"+m.group(1)+">";
|
||||
if(str2.toLowerCase().indexOf(tmp.toLowerCase())!=-1)//对应于sourceforge
|
||||
return true;
|
||||
if(str2.toLowerCase().indexOf(tmp2.toLowerCase())!=-1)//对应于openhub
|
||||
return true;
|
||||
if(str2.toLowerCase().indexOf(tmp3.toLowerCase())!=-1)//对应于freecode
|
||||
return true;
|
||||
if(str2.toLowerCase().equals(m.group(1).toLowerCase()))//对应于apahce
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 存在（）的将括号中的信息提取出来 去除常用词后得到另外一个linkName
|
||||
|
|
@ -95,36 +59,23 @@ public class StringHandler {
|
|||
List<String> result = new ArrayList<String>();
|
||||
int index1 = -1;
|
||||
int index2 = -1;
|
||||
int index_last1 = -1;
|
||||
int index_last2 = -1;
|
||||
String strInBracket = null;
|
||||
String strInBracket2 = null;
|
||||
|
||||
index1 = str.indexOf("(");
|
||||
index2 = str.indexOf(")");
|
||||
index_last1 = str.lastIndexOf("(");
|
||||
if(index1 > 0){
|
||||
result.add(str.substring(0,index1).trim());
|
||||
if(index2 > index1){
|
||||
strInBracket = str.substring(index1 + 1, index2);
|
||||
}
|
||||
if(index1 != index_last1){
|
||||
index_last2 = str.lastIndexOf(")");
|
||||
strInBracket2 = str.substring(index_last1 + 1, index_last2);
|
||||
}
|
||||
}else{
|
||||
index1 = str.indexOf("（");
|
||||
index2 = str.indexOf("）");
|
||||
index_last1 = str.lastIndexOf("（");
|
||||
if(index1 > 0){
|
||||
result.add(str.substring(0,index1).trim());
|
||||
if(index2 > index1){
|
||||
strInBracket = str.substring(index1 + 1, index2);
|
||||
}
|
||||
if(index1 != index_last1){
|
||||
index_last2 = str.lastIndexOf(")");
|
||||
strInBracket2 = str.substring(index_last1 + 1, index_last2);
|
||||
}
|
||||
}else{
|
||||
index1 = str.indexOf("[");
|
||||
index2 = str.indexOf("]");
|
||||
|
|
@ -159,9 +110,6 @@ public class StringHandler {
|
|||
if(strInBracket != null){
|
||||
result.add(strInBracket.trim());//暂时没有对括号中的数据进行常用词排除处理
|
||||
}
|
||||
if(strInBracket2 != null){
|
||||
result.add(strInBracket2.trim());//暂时没有对括号中的数据进行常用词排除处理
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
@ -235,22 +183,6 @@ public class StringHandler {
|
|||
|
||||
}
|
||||
|
||||
public static boolean getFullName(String shortName, String synonym,int a){
|
||||
|
||||
//查看当前的同义词是不是含不含有全称
|
||||
String[] words = synonym.toLowerCase().split(" ");
|
||||
String shortenWord = "";
|
||||
for(int i = 0; i < words.length; i++){
|
||||
String word = words[i];
|
||||
String tmp = word.substring(0, 1);//取出第一个英文字符
|
||||
shortenWord += tmp;
|
||||
}
|
||||
|
||||
if(shortenWord.toLowerCase().equals(shortName.toLowerCase()))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
//查看若干个char有没有在list中连续单词的首字母出现 如果出现返回连续出现的单词组成的字符串
|
||||
public static String getFullName(String shortName, String synonym){
|
||||
|
|
@ -295,84 +227,6 @@ public class StringHandler {
|
|||
return fullName;
|
||||
}
|
||||
|
||||
public static boolean isFullOrShort(String str1,String str2)
|
||||
{
|
||||
if(getFullName(str1,str2,1) || getFullName(str2, str1,1))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static boolean specialIsFullOrShort(String shortName, String fullName)
|
||||
{
|
||||
System.out.println("进入了特殊别名提取函数");
|
||||
//首先获取两个字符串中单词的
|
||||
String[] words_fullName= fullName.toLowerCase().split(" ");
|
||||
String[] words_shortName = shortName.toLowerCase().split(" ");
|
||||
|
||||
if(words_fullName.length==1 && words_shortName.length==1)
|
||||
{
|
||||
System.out.println("两者长度都大于1");
|
||||
return false;
|
||||
}
|
||||
|
||||
if(words_shortName.length == 1 && words_fullName.length > 1 )
|
||||
{
|
||||
System.out.println("fullName的长度大于1");
|
||||
if(specialIsFullOrShorDetail(shortName,fullName))
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
System.out.println("shortName长度大于1");
|
||||
if(specialIsFullOrShorDetail(fullName,shortName))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
public static boolean specialIsFullOrShorDetail(String shortName, String fullName)
|
||||
{
|
||||
//首先获取两个字符串中单词的个数
|
||||
String[] words_fullName = fullName.toLowerCase().split(" ");
|
||||
//String[] words_shortName = shortName.toLowerCase().split(" ");
|
||||
|
||||
int positionFlag = -1;
|
||||
int matchCount=0;
|
||||
int tmpFlag;
|
||||
|
||||
//要求全写中各个单词的首字母必须在缩写中出现而且是顺序出现或最多有一个不出现在缩写中。
|
||||
for(int i = 0; i < words_fullName.length; i++){
|
||||
String word = words_fullName[i].toLowerCase();
|
||||
String tmp = word.substring(0, 1);//取出第一个英文字符
|
||||
if(shortName.toLowerCase().contains(tmp))
|
||||
{
|
||||
System.out.println("进入了详细字符串比对函数");
|
||||
// tmpFlag = shortName.indexOf(tmp);
|
||||
// if(tmpFlag > positionFlag)
|
||||
// {
|
||||
matchCount++;
|
||||
// positionFlag = tmpFlag;
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
如果满足全写中各个单词的首字母都在缩写中顺序出现就接着
|
||||
判断缩写中所有字幕是否都在写中出现
|
||||
*/
|
||||
if(matchCount > words_fullName.length-2)
|
||||
{
|
||||
System.out.println("匹配个数与全写中单词个数相等");
|
||||
if(isAllContained(shortName,fullName))
|
||||
return true;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
//查看若干个char有没有在list中连续单词的首字母出现 如果出现返回连续出现的单词组成的字符串
|
||||
public static String getFullName(String shortName, List<String> list){
|
||||
String fullName = "";
|
||||
|
|
@ -392,7 +246,6 @@ public class StringHandler {
|
|||
// }
|
||||
}
|
||||
//判断shortName是不是shortenWord中的一部分
|
||||
shortenWord = shortenWord.toLowerCase();
|
||||
int index_start = shortenWord.indexOf(shortName);
|
||||
int index_end = index_start + shortName.length()-1;
|
||||
if(index_start >= 0){
|
||||
|
|
@ -415,33 +268,6 @@ public class StringHandler {
|
|||
return fullName;
|
||||
}
|
||||
|
||||
//判断是否shortName中所有字符都在fullName中出现，是，则返回真；否，则返回false
|
||||
private static boolean isAllContained(String shortName, String fullName) {
|
||||
// TODO Auto-generated method stub
|
||||
int matchCount =0;
|
||||
// for(int i=0;i<shortName.length();i++)
|
||||
// if(fullName.toLowerCase().contains(shortName.substring(i, i+1)))
|
||||
// matchCount++;
|
||||
//
|
||||
// if(matchCount > shortName.length()-2)
|
||||
// return true;
|
||||
int matchPosition;
|
||||
for(int j=0;j<shortName.length();j++)
|
||||
{
|
||||
matchPosition = fullName.toLowerCase().indexOf(shortName.substring(j,j+1).toLowerCase());
|
||||
if(matchPosition > -1)
|
||||
{
|
||||
fullName = fullName.substring(matchPosition);
|
||||
matchCount ++;
|
||||
}
|
||||
}
|
||||
|
||||
if(matchCount > shortName.length()-2)
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
//查看字符串中 在后面的list里面有没有首字母缩写的结果
|
||||
public static String getShortName(String fullName, List<String> list){
|
||||
//判断fullName是不是全英文的形式 如果不是 返回""
|
||||
|
|
@ -459,7 +285,7 @@ public class StringHandler {
|
|||
shortName += firstLetter;
|
||||
}
|
||||
for(String str:list){
|
||||
if(str.toLowerCase().equals(shortName)|| str.toLowerCase().indexOf(shortName)!=-1)
|
||||
if(str.equals(shortName)|| str.indexOf(shortName)!=-1)
|
||||
return str;//表示描述信息中出现了缩写形式
|
||||
}
|
||||
return "";
|
||||
|
|
|
|||
|
|
@ -104,41 +104,26 @@ public class StringHandler_ProjectName {
|
|||
* 提取项目名括号中的描述信息 如果没有括号 返回""
|
||||
*/
|
||||
public static String getBracket(String name){
|
||||
|
||||
int index1 = name.lastIndexOf("(");
|
||||
int index2 = -1;
|
||||
String inside = null;
|
||||
int index2;
|
||||
int index1 = name.indexOf("(");
|
||||
String inside = "";
|
||||
if(index1 > 0){
|
||||
index2 = name.lastIndexOf(")");
|
||||
index2 = name.indexOf(")");
|
||||
if(index2 > index1)
|
||||
inside = name.substring(index1 + 1, index2);//抽取括号内部的信息
|
||||
else{
|
||||
index2 = name.length();
|
||||
inside = name.substring(index1 + 1, index2);
|
||||
}
|
||||
}else{
|
||||
index1 = name.lastIndexOf("（");
|
||||
index1 = name.indexOf("（");
|
||||
if(index1 > 0){
|
||||
index2 = name.lastIndexOf("）");
|
||||
index2 = name.indexOf("）");
|
||||
if(index2 > index1)
|
||||
inside = name.substring(index1 + 1, index2);
|
||||
}
|
||||
else{
|
||||
index1 = name.lastIndexOf("[");
|
||||
if(index1 > 0){
|
||||
index2 = name.lastIndexOf("]");
|
||||
if(index2 > index1)
|
||||
inside = name.substring(index1 + 1, index2);
|
||||
}else{
|
||||
index1 = name.lastIndexOf("{");
|
||||
if(index1 > 0){
|
||||
index2 = name.lastIndexOf("}");
|
||||
if(index2 > index1)
|
||||
inside = name.substring(index1 + 1, index2);
|
||||
}else{
|
||||
index1 = name.lastIndexOf("【");
|
||||
if(index1 > 0){
|
||||
index2 = name.lastIndexOf("】");
|
||||
if(index2 > index1)
|
||||
inside = name.substring(index1 + 1, index2);
|
||||
}
|
||||
}
|
||||
else{
|
||||
index2 = name.length();
|
||||
inside = name.substring(index1 + 1, index2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -149,12 +134,12 @@ public class StringHandler_ProjectName {
|
|||
* 提取项目名中除去公司名的信息 作为别名
|
||||
*/
|
||||
public static String getProjectWithoutComName(String name){
|
||||
String result = name;
|
||||
String result = "";
|
||||
//读取需要去除的公司名信息
|
||||
List<String> companyList = FileReader.read("./files/companyName.txt");
|
||||
for(String company : companyList){
|
||||
name = name.trim();
|
||||
int index = name.toLowerCase().indexOf(company);
|
||||
name = name.trim().toLowerCase();
|
||||
int index = name.indexOf(company);
|
||||
if(index == 0){
|
||||
//表示找到了子字符串 并且在开头位置
|
||||
name = name.substring(company.length());
|
||||
|
|
|
|||
|
|
@ -20,10 +20,8 @@ import org.springframework.stereotype.Component;
|
|||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.ossean.dao.DBDest;
|
||||
import com.ossean.dao.DBSource;
|
||||
import com.ossean.dao.GatherDao;
|
||||
import com.ossean.dao.TransferPrjDao;
|
||||
import com.ossean.databaseDest.DBDest;
|
||||
import com.ossean.databaseSource.DBSource;
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.model.OpenSourceProject;
|
||||
|
|
@ -39,23 +37,19 @@ public class TransferProjectsUtil {
|
|||
private DBSource dbSource;
|
||||
@Resource
|
||||
private DBDest dbDest;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
@Resource
|
||||
private TransferPrjDao transferPrjDao;
|
||||
|
||||
@Qualifier("mergeProjectsUtil")
|
||||
@Autowired
|
||||
private MergeProjectsUtil mergeProjectsUtil;
|
||||
|
||||
private String gatherProjectTableName = "gather_projects_test";
|
||||
private String openSourceProjectTableName = "open_source_projects_17";
|
||||
private String tagTableName = "tags_17";
|
||||
private String taggingTableName = "taggings_17";
|
||||
private String eddRelationTableName = "edd_relations_17";
|
||||
private String pointerTableName = "edd_pointers_17";
|
||||
private String sourceTableName = "edd_relations_17";
|
||||
private String targetTableName = "open_source_projects_17";
|
||||
|
||||
private String synonymTableName = "synonyms_17";
|
||||
private String synonymmingTableName = "synonymmings_17";
|
||||
private String gatherProjectTableName = "gather_projects";
|
||||
private String openSourceProjectTableName = "open_source_projects";
|
||||
private String tagTableName = "tags";
|
||||
private String taggingTableName = "taggings";
|
||||
private String eddRelationTableName = "edd_relations";
|
||||
private String pointerTableName = "edd_pointers";
|
||||
private String sourceTableName = "edd_relations";
|
||||
private String targetTableName = "open_source_projects";
|
||||
|
||||
private static int NUMBER_OF_TAGS = 50;
|
||||
private static int NUMBER_OF_CATEGORIES = 5;
|
||||
|
|
@ -91,7 +85,7 @@ public class TransferProjectsUtil {
|
|||
boolean error_lost = false;
|
||||
for(int i = 0; i < idsArray.length; i++){
|
||||
String id = idsArray[i];
|
||||
GatherProjectsModel p = gatherDao.selectGPMById(gatherProjectTableName, Integer.parseInt(id));
|
||||
GatherProjectsModel p = dbSource.getGatherProjectById(gatherProjectTableName, Integer.parseInt(id));
|
||||
if (p != null)//防止因为数据不一致导致的没有取到相应的数据
|
||||
gatherProjectList.add(p);
|
||||
else{
|
||||
|
|
@ -108,7 +102,7 @@ public class TransferProjectsUtil {
|
|||
gather_projects_ids_new = gather_projects_ids_new + id + ",";
|
||||
if(gather_projects_ids_new.length() > 1){
|
||||
//更新gather_projects_ids字段
|
||||
transferPrjDao.updateEddRelationGatherProjectsIds(eddRelationTableName, relation.getId(), gather_projects_ids_new);
|
||||
dbSource.updateEddRelationGatherProjectsIds(eddRelationTableName, relation.getId(), gather_projects_ids_new);
|
||||
}else{
|
||||
//删除该relation记录
|
||||
dbSource.deleteEddRelationsItem(eddRelationTableName, relation.getId());
|
||||
|
|
@ -195,25 +189,11 @@ public class TransferProjectsUtil {
|
|||
for(GatherProjectsModel gatherProject:gatherProjectList){
|
||||
tags = mergeTextWithoutSource(tags, gatherProject.getTags(), NUMBER_OF_TAGS);
|
||||
|
||||
// //处理同义词 获取每个项目最本质的同义词作为结果
|
||||
// List<String> synonyms_tmp = getSynonyms(gatherProject);
|
||||
// for(String s:synonyms_tmp){
|
||||
// if(!s.equals(hottestProject.getName().toLowerCase()))
|
||||
// synonyms = synonyms + "<" + s + ">,"; //如果与项目名同名则不加入
|
||||
// }
|
||||
|
||||
List<Integer> synonymList = dbSource.getSynonymmingList(synonymmingTableName,gatherProject.getId());
|
||||
for(int i=0;i<synonymList.size();i++){
|
||||
int synonym_id = synonymList.get(i);
|
||||
// int relate_count = dbSource.selectSynonymmingCount(synonymmingTableName, synonym_id);
|
||||
// if(relate_count>=2)
|
||||
// continue;
|
||||
String synonym_name = dbSource.getSynonymNameById(synonymTableName,synonym_id);
|
||||
if(dbDest.selectPrjNumFromOSP(targetTableName,'<'+synonym_name+'>') >=1)
|
||||
continue;
|
||||
if(!synonym_name.toLowerCase().equals(hottestProject.getName().toLowerCase())){
|
||||
synonyms = synonyms + "<" + synonym_name + ">,"; //如果与项目名同名则不加入
|
||||
}
|
||||
//处理同义词 获取每个项目最本质的同义词作为结果
|
||||
List<String> synonyms_tmp = getSynonyms(gatherProject);
|
||||
for(String s:synonyms_tmp){
|
||||
if(!s.equals(hottestProject.getName().toLowerCase()))
|
||||
synonyms = synonyms + "<" + s + ">,"; //如果与项目名同名则不加入
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,738 +0,0 @@
|
|||
package com.ossean.util;
|
||||
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.transaction.annotation.Propagation;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
import com.ossean.TableName;
|
||||
import com.ossean.dao.DBDest;
|
||||
import com.ossean.dao.DBSource;
|
||||
import com.ossean.dao.GatherDao;
|
||||
import com.ossean.dao.TransferPrjDao;
|
||||
import com.ossean.model.EddRelations;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.model.OpenSourceProject;
|
||||
import com.ossean.model.Synonymmings;
|
||||
import com.ossean.model.Synonyms;
|
||||
import com.ossean.model.Taggings;
|
||||
|
||||
@Component("transferProjectsUtil2")
|
||||
public class TransferProjectsUtil2 {
|
||||
Logger logger = Logger.getLogger(this.getClass());
|
||||
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
@Resource
|
||||
private DBDest dbDest;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
@Resource
|
||||
private TransferPrjDao transferPrjDao;
|
||||
|
||||
private String gatherProjectTableName = TableName.gatherProjectsTableName;
|
||||
private String openSourceProjectTableName = TableName.openSourceProjectsTableName;
|
||||
private String tagTableName = TableName.tagsTableName;
|
||||
private String taggingTableName = TableName.taggingsTableName;
|
||||
private String eddRelationTableName = TableName.eddRelationTableName;
|
||||
private String pointerTableName = TableName.pointerTableName;
|
||||
private String sourceTableName = TableName.eddRelationTableName;
|
||||
private String targetTableName = TableName.openSourceProjectsTableName;
|
||||
private String synonymTableName = TableName.synonymsTableName;
|
||||
|
||||
private static int NUMBER_OF_TAGS = 50;
|
||||
private static int NUMBER_OF_CATEGORIES = 5;
|
||||
private static int NUMBER_OF_LANGUAGES = 5;
|
||||
|
||||
|
||||
public static String getTargetTable(int ospId){
|
||||
String targetTableName = "";
|
||||
if(ospId >= 770000){
|
||||
targetTableName = "relative_memo_to_open_source_projects_70";
|
||||
}
|
||||
else{
|
||||
int a = 1 + ospId/11000;
|
||||
targetTableName = "relative_memo_to_open_source_projects_" + a;
|
||||
}
|
||||
|
||||
return targetTableName;
|
||||
}
|
||||
|
||||
|
||||
//将GatherProject的list合并为OpenSourceProject
|
||||
//需要将最热门项目存储到第一个下标的位置
|
||||
public OpenSourceProject mergeGatherProjectsToOpenSourceProject(List<GatherProjectsModel> gatherProjectList){
|
||||
if(gatherProjectList.size() == 0 || gatherProjectList == null)
|
||||
return null;
|
||||
OpenSourceProject result = new OpenSourceProject();
|
||||
GatherProjectsModel hottestProject = getHottestProject(gatherProjectList);
|
||||
if(hottestProject == null){
|
||||
//表示没有找到最热门的项目 只有freecode项目
|
||||
return null;
|
||||
}
|
||||
int id = hottestProject.getId();
|
||||
String name = hottestProject.getName();
|
||||
String description = null;
|
||||
String url = null;
|
||||
String url_md5 = null;
|
||||
String language = null;
|
||||
String category = null;
|
||||
String source = mergeSource(gatherProjectList, hottestProject.getId());
|
||||
String tags = null;
|
||||
String tags_for_search = null;
|
||||
String synonyms = "";
|
||||
String license = hottestProject.getLicense();
|
||||
String homepage = hottestProject.getHomepage();
|
||||
String updated_time = hottestProject.getUpdated_time();
|
||||
String extracted_time = hottestProject.getExtracted_time();
|
||||
String created_time = hottestProject.getCreated_time();
|
||||
/**
|
||||
* 将gatherProjectList列表中的最热门项目放在第一个下标的位置
|
||||
*/
|
||||
gatherProjectList = putHottestInHead(gatherProjectList, id);
|
||||
|
||||
List<String> synonyms_tmp = new ArrayList<String>();
|
||||
for(GatherProjectsModel gatherProject:gatherProjectList){
|
||||
tags = mergeTextWithoutSource(tags, gatherProject.getTags(), NUMBER_OF_TAGS);
|
||||
synonyms_tmp.addAll(dbSource.getSynonymByPrjIdForTransfer(synonymTableName, gatherProject.getId()));//处理同义词
|
||||
|
||||
if(!(("FreeCode".toLowerCase()).equals(gatherProject.getSource().toLowerCase()))){
|
||||
//表示如果是FreeCode社区中的项目 只将tags属性合并 不进行下面的步骤
|
||||
description = mergeTextWithSource(description, gatherProject.getDescription(), gatherProject.getSource());
|
||||
url = mergeTextWithSource(url, gatherProject.getUrl(), gatherProject.getSource());
|
||||
url_md5 = mergeTextWithSource(url_md5, gatherProject.getUrl_md5(), gatherProject.getSource());
|
||||
language = mergeTextWithoutSource(language, gatherProject.getLanguage(),NUMBER_OF_LANGUAGES);
|
||||
category = mergeTextWithoutSource(category, gatherProject.getCategory(),NUMBER_OF_CATEGORIES);
|
||||
}
|
||||
}
|
||||
synonyms_tmp = removeDupStr(synonyms_tmp);
|
||||
for(String s:synonyms_tmp){
|
||||
if(!s.toLowerCase().equals(hottestProject.getName().toLowerCase()))
|
||||
synonyms = synonyms + "<" + s + ">,"; //如果与项目名同名则不加入
|
||||
}
|
||||
if("".equals(synonyms))
|
||||
synonyms = null;
|
||||
result.setId(id);
|
||||
result.setName(name);
|
||||
result.setDescription(description);
|
||||
result.setUrl(url);
|
||||
result.setUrl_md5(url_md5);
|
||||
result.setLanguage(language);
|
||||
result.setCategory(category);
|
||||
result.setSource(source);
|
||||
result.setTags(tags);
|
||||
result.setCreated_time(created_time);
|
||||
result.setUpdated_time(updated_time);
|
||||
result.setTags_for_search(tags_for_search);
|
||||
result.setSynonyms(synonyms);
|
||||
result.setLicense(license);
|
||||
result.setHomepage(homepage);
|
||||
result.setExtracted_time(extracted_time);
|
||||
result.setUpdate_mark(1);//表示是待更新的条目
|
||||
if(("FreeCode".toLowerCase()).equals(result.getSource().toLowerCase()) || result.getName() == null || result.getDescription() == null)
|
||||
return null;
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理一个edd_relation
|
||||
* @param eddRelation
|
||||
* @return
|
||||
*/
|
||||
@Transactional(propagation=Propagation.REQUIRED)
|
||||
public void handleOneRelation(EddRelations relation){
|
||||
//logger.info("处理edd_relations表数据，id：" + relation.getId());
|
||||
List<GatherProjectsModel> gatherProjectList = new ArrayList<GatherProjectsModel>();//最热门项目要放在第一个下标的位置
|
||||
String gather_projects_ids = relation.getGather_projects_ids();
|
||||
gather_projects_ids = gather_projects_ids.substring(1, gather_projects_ids.length() - 1);
|
||||
String[] idsArray = gather_projects_ids.split(",");
|
||||
boolean error_lost = false;
|
||||
for(int i = 0; i < idsArray.length; i++){
|
||||
String id = idsArray[i];
|
||||
GatherProjectsModel p = gatherDao.selectGPMById(gatherProjectTableName, Integer.parseInt(id));
|
||||
if (p != null)//防止因为数据不一致导致的没有取到相应的数据
|
||||
gatherProjectList.add(p);
|
||||
else{
|
||||
//删除没有的id
|
||||
error_lost = true;
|
||||
idsArray[i] = null;
|
||||
}
|
||||
}
|
||||
if(error_lost){
|
||||
//需要更新gather_projects_ids字段
|
||||
String gather_projects_ids_new = ",";
|
||||
for(String id:idsArray)
|
||||
if(id != null)
|
||||
gather_projects_ids_new = gather_projects_ids_new + id + ",";
|
||||
if(gather_projects_ids_new.length() > 1){
|
||||
//更新gather_projects_ids字段
|
||||
transferPrjDao.updateEddRelationGatherProjectsIds(eddRelationTableName, relation.getId(), gather_projects_ids_new);
|
||||
}else{
|
||||
//删除该relation记录
|
||||
dbSource.deleteEddRelationsItem(eddRelationTableName, relation.getId());
|
||||
}
|
||||
}
|
||||
OpenSourceProject osp = mergeGatherProjectsToOpenSourceProject(gatherProjectList);
|
||||
if(osp != null){
|
||||
boolean sameAlready = hasSameOsp(gather_projects_ids, osp);
|
||||
if(!sameAlready){
|
||||
dbDest.insertOsp(openSourceProjectTableName, osp);//将数据插入open_source_projects表中
|
||||
int osp_id = osp.getId();
|
||||
String tags = osp.getTags();//tag分离
|
||||
if(tags == null){//表示该项目没有标签
|
||||
return;
|
||||
}
|
||||
String[] tagArray = tags.split(",");
|
||||
for(String tag:tagArray){
|
||||
tag = tag.substring(1, tag.length() - 1).toLowerCase();//去掉两边的尖括号 并转换为小写
|
||||
dbDest.insertTag(tagTableName, tag);//ignore方式插入该项目的标签
|
||||
int tag_id = dbDest.selectTagIdByName(tagTableName, tag);
|
||||
Taggings tagging = new Taggings();
|
||||
tagging.setTag_id(tag_id);
|
||||
tagging.setTaggable_id(osp_id);
|
||||
tagging.setTaggable_type("OpenSourceProject");
|
||||
tagging.setContext("tags");
|
||||
tagging.setCreated_at(DateHandler.getNow());
|
||||
tagging.setDisagree_num(50);
|
||||
try {
|
||||
dbDest.insertTagging(taggingTableName, tagging);//将Taggings对象存入数据库中
|
||||
} catch (Exception e) {
|
||||
//logger.info("osp_id has been removed");//在插入同义词时 open_source_projects中的记录已经被删除掉了
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
/**
|
||||
* 获取合适的id 用最热门项目的id表示
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
public int getSuitableId(List<GatherProjectsModel> gatherProjectList){
|
||||
int smallestId = Integer.MAX_VALUE;
|
||||
for(GatherProjectsModel model:gatherProjectList){
|
||||
if(model.getId() < smallestId)
|
||||
smallestId = model.getId();
|
||||
}
|
||||
return smallestId;
|
||||
}
|
||||
//获取gatherProjectsList中最合适的项目名
|
||||
public String getSuitableName(List<GatherProjectsModel> gatherProjectList){
|
||||
//获取每个项目的source 排序为：apache[0]、oschina[1]、openhub[2]、sourceforge[3]
|
||||
GatherProjectsModel hottestProject = getHottestProject(gatherProjectList);
|
||||
return hottestProject.getName();
|
||||
}
|
||||
//获取最热门项目
|
||||
public GatherProjectsModel getHottestProject(List<GatherProjectsModel> gatherProjectList){
|
||||
int head = Integer.MAX_VALUE;
|
||||
GatherProjectsModel result = null;
|
||||
for(GatherProjectsModel project:gatherProjectList){
|
||||
String source = project.getSource();
|
||||
if(source == null)
|
||||
source = "";
|
||||
int tmp_value = Integer.MAX_VALUE;
|
||||
if(source.toLowerCase().equals("apache"))
|
||||
tmp_value = 0;
|
||||
else if(source.toLowerCase().equals("oschina"))
|
||||
tmp_value = 1;
|
||||
else if(source.toLowerCase().equals("openhub"))
|
||||
tmp_value = 2;
|
||||
else if(source.toLowerCase().equals("sourceforge"))
|
||||
tmp_value = 3;
|
||||
else
|
||||
tmp_value = Integer.MAX_VALUE;
|
||||
if(head > tmp_value){
|
||||
head = tmp_value;
|
||||
result = project;
|
||||
}else{
|
||||
//result = null;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
public List<String> removeDupStr(List<String> array){
|
||||
List<String> list = new ArrayList<String>();
|
||||
Set<String> set = new HashSet<String>();
|
||||
for(String s : array){
|
||||
if(set.add(s.toLowerCase().trim())){
|
||||
list.add(s);
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
//合并没有项目源区分的字段 第三个参数num表示返回所有值里面的多少个 -1表示全部返回
|
||||
public String mergeTextWithoutSource(String tag1, String tag2, int num){
|
||||
if(tag1 == null)
|
||||
tag1 = "";
|
||||
if(tag2 == null)
|
||||
tag2 = "";
|
||||
String result = "";
|
||||
tag1 = tag1.trim();
|
||||
tag2 = tag2.trim();
|
||||
String[] tagArray1 = tag1.split(",");
|
||||
String[] tagArray2 = tag2.split(",");
|
||||
Map<String, Integer> map = new HashMap<String, Integer>();
|
||||
for(String tag:tagArray1)
|
||||
map.put(tag, 1);
|
||||
for(String tag:tagArray2)
|
||||
map.put(tag, 1);
|
||||
//将map转换为List
|
||||
Set<String> keys = map.keySet();
|
||||
Iterator<String> it = keys.iterator();
|
||||
int count = 0; //用于计数 记录多少个值
|
||||
while(it.hasNext()){
|
||||
String key = it.next();
|
||||
if(!"".equals(key)){
|
||||
//如果是空字符串 不添加进去
|
||||
result += key + ",";
|
||||
count++;
|
||||
}
|
||||
if(count >= num && num >=0) //为负值表示保留所有结果
|
||||
break;
|
||||
}
|
||||
if(result.length() == 0)
|
||||
return null;
|
||||
return result.substring(0, result.length() - 1);
|
||||
}
|
||||
//合并有项目源区分的字段
|
||||
public String mergeTextWithSource(String text1, String text2, String source){
|
||||
//text2是新增的 text1是以前的
|
||||
if(text2 == null && text1 != null)
|
||||
return text1;
|
||||
if(text2 != null && text1 == null){
|
||||
return source + "|:|" + text2.trim();
|
||||
}
|
||||
if(text1 != null && text1.length() != 0){
|
||||
return text1 + "|,|" + source + "|:|" + text2.trim();
|
||||
}
|
||||
return text1;
|
||||
}
|
||||
//合并数字类的记录
|
||||
public int mergeInteger(Integer integer1, Integer integer2){
|
||||
if(integer1 == null)
|
||||
return integer2;
|
||||
if(integer2 == null)
|
||||
return integer1;
|
||||
return integer1 + integer2;
|
||||
}
|
||||
//合并日期类记录
|
||||
public String mergeTime(String time1, String time2){
|
||||
if(time2 == null)
|
||||
return time1;
|
||||
if(time1 == null)
|
||||
return time2;
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
try {
|
||||
Date date1 = sdf.parse(time1);
|
||||
Date date2 = sdf.parse(time2);
|
||||
if(date1.before(date2)){
|
||||
return time1;
|
||||
}
|
||||
return time2;
|
||||
} catch (ParseException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
return time1;//如果出现错误 原来的时间不变
|
||||
}
|
||||
}
|
||||
|
||||
//合并项目来源
|
||||
public String mergeSource(String source1, String source2){
|
||||
if (source2 == null)
|
||||
return source1;
|
||||
if (source1 == null)
|
||||
return source2;
|
||||
String result = "";
|
||||
Map<String, Integer> map = new HashMap<String, Integer>();
|
||||
String[] array1 = source1.split(",");
|
||||
for(String source:array1)
|
||||
map.put(source, 1);
|
||||
String[] array2 = source2.split(",");
|
||||
for(String source:array2)
|
||||
map.put(source, 1);
|
||||
Set<String> keys = map.keySet();
|
||||
Iterator<String> it = keys.iterator();
|
||||
while(it.hasNext()){
|
||||
String key = it.next();
|
||||
result = result + key + ",";
|
||||
}
|
||||
result = result.substring(0, result.length() - 1);
|
||||
return result;
|
||||
}
|
||||
|
||||
//更新的合并项目来源函数
|
||||
public String mergeSource(List<GatherProjectsModel> gatherProjectList, int hottest_id){
|
||||
String result = "";
|
||||
for(int i = 0; i < gatherProjectList.size(); i++){
|
||||
GatherProjectsModel model = gatherProjectList.get(i);
|
||||
int id = model.getId();
|
||||
String source = model.getSource();
|
||||
//判断是不是freecode
|
||||
if("freecode".equals(source.toLowerCase())){
|
||||
continue;
|
||||
}
|
||||
if(id == hottest_id){
|
||||
result = source + "," + result;
|
||||
}else{
|
||||
result = result + source + ",";
|
||||
}
|
||||
}
|
||||
result = result.substring(0, result.length() - 1);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 查看当前新形成的osp有没有每个id对应的osp_former 使得他们的name tag synonyms属性不相同
|
||||
* 如果有不相同 则删除原来的记录 并插入一条新的记录
|
||||
* 如果全部相同 则不进行后续操作
|
||||
*/
|
||||
public boolean hasSameOsp(String gather_projects_ids, OpenSourceProject osp){
|
||||
boolean result = false;
|
||||
String[] idArray = gather_projects_ids.split(",");
|
||||
for(String id_str:idArray){
|
||||
//查找对应osp记录
|
||||
int id = Integer.parseInt(id_str);
|
||||
OpenSourceProject osp_former = dbDest.selectOpenSourceProjectsItem(openSourceProjectTableName, id);
|
||||
if(osp_former != null){
|
||||
if(sameName(osp_former, osp) && sameTags(osp_former, osp) && sameSynonyms(osp_former, osp) && result == false){
|
||||
result = true;
|
||||
continue;//认为项目相同 不替换
|
||||
}else{
|
||||
//删除osp_former
|
||||
dbDest.deleteOpenSourceProjectsItem(openSourceProjectTableName,id);
|
||||
dbSource.deleteTaggingsByTaggableId(taggingTableName, id); //由于没有级联删除 因此需要手动删除
|
||||
dbDest.deleteMatchResult(id, getTargetTable(id));
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 比较两个osp对象的姓名是否相同
|
||||
*/
|
||||
public boolean sameName(OpenSourceProject osp_former, OpenSourceProject osp){
|
||||
if((osp_former.getName().toLowerCase()).equals(osp.getName().toLowerCase()))
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 比较两个osp对象的标签是否相同
|
||||
*/
|
||||
public boolean sameTags(OpenSourceProject osp_former, OpenSourceProject osp){
|
||||
String tags_former = osp_former.getTags();
|
||||
String tags = osp.getTags();
|
||||
if(tags_former == null && tags == null)
|
||||
return true;
|
||||
else if((tags_former != null && tags == null) || (tags_former == null && tags != null))
|
||||
return false;
|
||||
Map<String, Integer> tagMap_former = new HashMap<String, Integer>();
|
||||
Map<String, Integer> tagMap = new HashMap<String, Integer>();
|
||||
String[] tagArray_former = tags_former.split(",");
|
||||
String[] tagArray = tags.split(",");
|
||||
for(String tag:tagArray_former)
|
||||
tagMap_former.put(tag, 1);
|
||||
for(String tag:tagArray)
|
||||
tagMap.put(tag, 1);
|
||||
|
||||
for(String tag:tagArray_former)
|
||||
if(!tagMap.containsKey(tag))
|
||||
return false;
|
||||
|
||||
for(String tag:tagArray)
|
||||
if(!tagMap_former.containsKey(tag))
|
||||
return false;
|
||||
|
||||
//如果相互之间每个tag都存在 则证明两个tags属性完全相同
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 比较两个osp对象的同义词是否相同
|
||||
*/
|
||||
public boolean sameSynonyms(OpenSourceProject osp_former, OpenSourceProject osp){
|
||||
String synonyms_former = osp_former.getSynonyms();
|
||||
String synonyms = osp.getSynonyms();
|
||||
if(synonyms_former == null && synonyms == null)
|
||||
return true;
|
||||
else if((synonyms_former != null && synonyms == null) || (synonyms_former == null && synonyms != null))
|
||||
return false;
|
||||
Map<String, Integer> synonymMap_former = new HashMap<String, Integer>();
|
||||
Map<String, Integer> synonymMap = new HashMap<String, Integer>();
|
||||
String[] synonymArray_former = synonyms_former.split(",");
|
||||
String[] synonymArray = synonyms.split(",");
|
||||
for(String synonym:synonymArray_former)
|
||||
synonymMap_former.put(synonym, 1);
|
||||
for(String synonym:synonymArray)
|
||||
synonymMap.put(synonym, 1);
|
||||
|
||||
for(String synonym:synonymArray_former)
|
||||
if(!synonymMap.containsKey(synonym))
|
||||
return false;
|
||||
|
||||
for(String synonym:synonymArray)
|
||||
if(!synonymMap_former.containsKey(synonym))
|
||||
return false;
|
||||
|
||||
//如果相互之间每个synonym都存在 则证明两个synonyms属性完全相同
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 将gather_projects_ids转换为最热门项目id在前的
|
||||
*/
|
||||
public List<GatherProjectsModel> putHottestInHead(List<GatherProjectsModel> gatherProjectList, int hottest_id){
|
||||
for(int i = 0; i < gatherProjectList.size(); i++){
|
||||
GatherProjectsModel model = gatherProjectList.get(i);
|
||||
int id = model.getId();
|
||||
if(id == hottest_id){
|
||||
//删除当前下标的 再插入到最前面
|
||||
gatherProjectList.remove(i);
|
||||
gatherProjectList.add(0, model);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return gatherProjectList;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 与mergeProjects里面的函数区别为 不考虑数据库中已经提取出来的情况
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public List<String> getSynonyms(GatherProjectsModel model){
|
||||
List<String> result = new ArrayList<String>();
|
||||
String description = model.getDescription();
|
||||
if(description == null)
|
||||
return result;
|
||||
|
||||
model.setName(model.getName().trim());
|
||||
model.setDescription(model.getDescription().trim());
|
||||
//最终返回的同义词列表
|
||||
List<String> synonymsList = new ArrayList<String>();
|
||||
List<String> linkNameList = new ArrayList<String>();//用于存储所有提取出来的关联项目名
|
||||
List<String> keywords = FileReader.read("./files/language.txt");
|
||||
|
||||
//加入项目名含括号，加入括号中的内容
|
||||
String prjName = model.getName();
|
||||
String prjNameNoBracket = StringHandler_ProjectName.removeBracket(prjName);
|
||||
if(prjName.indexOf("(") != -1 || prjName.indexOf("（")!=-1){
|
||||
synonymsList.add(prjNameNoBracket);
|
||||
}
|
||||
|
||||
//提取项目名括号中的信息，不是项目类型
|
||||
String bracket = StringHandler_ProjectName.getBracket(model.getName());
|
||||
if(!"".equals(bracket)){
|
||||
bracket = RegexHandler.extractEngDecChiAndDot(bracket);//去除括号中信息的特殊符号
|
||||
if(!RegexHandler.onlySpecialChar(bracket)){
|
||||
if(!keywords.contains(bracket)){//括号中的信息不是语言类关键字词
|
||||
linkNameList.add(bracket);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//对项目名进行处理,提取项目名中去除公司名或基金组织的信息
|
||||
String projectNameWithoutComName = StringHandler_ProjectName.getProjectWithoutComName(model.getName());
|
||||
projectNameWithoutComName = StringHandler_ProjectName.removeBracket(projectNameWithoutComName);
|
||||
if(!"".equals(projectNameWithoutComName)){
|
||||
//表示匹配到了相应公司或基金委的项目
|
||||
synonymsList.add(projectNameWithoutComName);
|
||||
}
|
||||
|
||||
|
||||
if(description != null){
|
||||
|
||||
List<String> beWords = FileReader.read("./files/beVerb.txt");//读取be动词表
|
||||
description = StringHandler.getFirstSentence(description);
|
||||
// if (description.indexOf(":") != -1){//提取描述中冒号之前的信息
|
||||
// int indexColon = description.indexOf(":");
|
||||
// String beforeColon = description.substring(0, indexColon);
|
||||
// linkNameList.add(beforeColon.trim());
|
||||
// }
|
||||
|
||||
for(String beWord:beWords){
|
||||
|
||||
String linkName = StringHandler.findLinkName(description, beWord);//按系动词表优先级进行匹配
|
||||
if(linkName == null){//表示没有匹配到当前的系动词 进行下一个匹配
|
||||
continue;
|
||||
}
|
||||
//特征短语去噪
|
||||
//提取主副描述信息(括号中)
|
||||
List<String> linkNameResult = EDDHandler.getMainAndViceDescriptionByBracket(linkName);
|
||||
|
||||
//提取主副描述信息(known as+连词)
|
||||
linkNameResult = EDDHandler.getMainAndViceDescriptionByKnowAs(linkNameResult);
|
||||
//去除定语描述信息(逗号定语)
|
||||
linkNameResult = EDDHandler.removeComma(linkNameResult);
|
||||
//去除定语描述信息(定冠词The)
|
||||
linkNameResult = EDDHandler.removeDemonstrativeWords(linkNameResult);
|
||||
//去除从句描述信息
|
||||
linkNameResult = EDDHandler.removeArrtibutiveClause(linkNameResult);
|
||||
//去除指示代词
|
||||
linkNameResult = EDDHandler.removePronoun(linkNameResult);
|
||||
//去除项目常用词
|
||||
linkNameResult = EDDHandler.removeProject(linkNameResult);
|
||||
//去除指示代词和项目常用词两者的笛卡尔积
|
||||
linkNameResult = EDDHandler.removePPCombine(linkNameResult);
|
||||
//提取项目中
|
||||
linkNameResult = RegexHandler.extractEngDecChiAndDot(linkNameResult);
|
||||
//删除只有特殊字符的同义词
|
||||
linkNameResult = RegexHandler.removeOnlySpecial(linkNameResult);
|
||||
for(String name:linkNameResult){
|
||||
//System.out.println("be 动词之前的有： "+name);
|
||||
//be动词前不包括项目类型，且和项目名不相同
|
||||
if(!keywords.contains(name) && !name.equals(model.getName().toLowerCase())){
|
||||
if(model.getName().toLowerCase().indexOf(name) != -1 && !name.equals(prjNameNoBracket))
|
||||
synonymsList.add(name);
|
||||
else
|
||||
linkNameList.add(name);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
//result包含去掉公司信息、括号内的、be动词之前的
|
||||
result = new ArrayList(new HashSet(linkNameList));
|
||||
//System.out.println("result包含去掉公司信息、括号内的、be动词之前的;");
|
||||
//for (int i = 0;i<result.size();i++)
|
||||
//System.out.println("result: "+result.get(i));
|
||||
|
||||
String fullName = StringHandler.getFullName(prjNameNoBracket.toLowerCase(), result);
|
||||
if(!fullName.equals("")){
|
||||
synonymsList.add(fullName.toLowerCase());//表示项目名确实是缩写 并提取到了项目全名
|
||||
//System.out.println("项目的全称为： "+fullName);
|
||||
}
|
||||
|
||||
|
||||
String shortName = StringHandler.getShortName(prjNameNoBracket.toLowerCase(), result);
|
||||
if(!shortName.equals("")){
|
||||
synonymsList.add(shortName.toLowerCase());//表示项目名确实是全称 并提取到了项目缩写
|
||||
//System.out.println("项目的简称为： "+shortName);
|
||||
}
|
||||
|
||||
//同义词要和项目名称有一定的相似度
|
||||
for(int i = 0; i < result.size(); i++){
|
||||
String extract = "";
|
||||
String synonym = result.get(i);
|
||||
String[] words = synonym.split(" ");//按照空格进行分词
|
||||
for(String word:words){
|
||||
if(model.getName().indexOf(word) >= 0 ||model.getName().toLowerCase().indexOf(word) >= 0){//查看每个词在项目名中是否出现
|
||||
extract += word + " ";
|
||||
}
|
||||
}
|
||||
if(extract.length() !=0)
|
||||
extract = extract.substring(0, extract.length() - 1);
|
||||
//System.out.println("extract : "+extract);
|
||||
if(!"".equals(extract) && !keywords.contains(extract)){
|
||||
//提取的别名不和项目名重复
|
||||
if(!extract.toLowerCase().equals(model.getName().toLowerCase().trim()) && !extract.toLowerCase().trim().equals(prjNameNoBracket.toLowerCase().trim())){
|
||||
double similarDegree = SimilarDegree(extract,prjNameNoBracket);
|
||||
//System.out.println("字符串间的相似度为: "+similarDegree);
|
||||
if(similarDegree >= 0.4 && similarDegree <1.0){
|
||||
//System.out.println("别名在项目名中有出现： "+extract);
|
||||
synonymsList.add(extract);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
//添加项目名本身，用于利用别名寻找匹配项目时
|
||||
synonymsList.add(model.getName().toLowerCase());
|
||||
synonymsList = new ArrayList(new HashSet(synonymsList));
|
||||
//输出日志文件
|
||||
for(String s : synonymsList){
|
||||
//logger.info("gather_project:" + model.getId() + " gets synonyms:" + s + " by description");
|
||||
}
|
||||
//去掉重复元素
|
||||
return synonymsList;
|
||||
}
|
||||
public static double SimilarDegree(String strA, String strB){
|
||||
|
||||
String newStrA = removeSign(strA);
|
||||
String newStrB = removeSign(strB);
|
||||
int temp = Math.max(newStrA.length(), newStrB.length());
|
||||
int temp2 = longestCommonSubstring(newStrA, newStrB).length();
|
||||
return temp2 * 1.0 / temp;
|
||||
|
||||
}
|
||||
|
||||
private static String removeSign(String str) {
|
||||
StringBuffer sb = new StringBuffer();
|
||||
for (char item : str.toCharArray())
|
||||
if (charReg(item)){
|
||||
sb.append(item);
|
||||
}
|
||||
return sb.toString();
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static boolean charReg(char charValue) {
|
||||
|
||||
return (charValue >= 0x4E00 && charValue <= 0X9FA5)
|
||||
|| (charValue >= 'a' && charValue <= 'z')
|
||||
|| (charValue >= 'A' && charValue <= 'Z')
|
||||
|| (charValue >= '0' && charValue <= '9');
|
||||
|
||||
}
|
||||
|
||||
private static String longestCommonSubstring(String strA, String strB) {
|
||||
char[] chars_strA = strA.toCharArray();
|
||||
char[] chars_strB = strB.toCharArray();
|
||||
int m = chars_strA.length;
|
||||
int n = chars_strB.length;
|
||||
int[][] matrix = new int[m + 1][n + 1];
|
||||
for (int i = 1; i <= m; i++) {
|
||||
|
||||
for (int j = 1; j <= n; j++) {
|
||||
if (chars_strA[i - 1] == chars_strB[j - 1])
|
||||
matrix[i][j] = matrix[i - 1][j - 1] + 1;
|
||||
else
|
||||
matrix[i][j] = Math.max(matrix[i][j - 1], matrix[i - 1][j]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
char[] result = new char[matrix[m][n]];
|
||||
int currentIndex = result.length - 1;
|
||||
while (matrix[m][n] != 0) {
|
||||
if (matrix[m][n] == matrix[m][n - 1])
|
||||
n--;
|
||||
else if (matrix[m][n] == matrix[m - 1][n])
|
||||
m--;
|
||||
else {
|
||||
result[currentIndex] = chars_strA[m - 1];
|
||||
currentIndex--;
|
||||
n--;
|
||||
m--;
|
||||
}
|
||||
}
|
||||
return new String(result);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -29,16 +29,17 @@
|
|||
<property name="dataSource" ref="dataSourceOne" />
|
||||
</bean>
|
||||
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
|
||||
<property name="basePackage" value="com.ossean.dao" />
|
||||
<property name="basePackage" value="com.ossean" />
|
||||
<property name="sqlSessionFactory" ref="sqlSessionFactoryOne"></property>
|
||||
</bean>
|
||||
|
||||
|
||||
<bean id="dataSourceTwo" class="org.apache.commons.dbcp.BasicDataSource"
|
||||
<!--DestData -->
|
||||
<!-- <bean id="dataSourceTwo" class="org.apache.commons.dbcp.BasicDataSource"
|
||||
destroy-method="close">
|
||||
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
|
||||
<property name="url"
|
||||
value="jdbc:mysql://localhost:3306/ossean?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
value="jdbc:mysql://localhost:3306/ossean_show?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="root" />
|
||||
<property name="password" value="123456" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
|
|
@ -48,9 +49,9 @@
|
|||
<property name="dataSource" ref="dataSourceTwo" />
|
||||
</bean>
|
||||
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
|
||||
<property name="basePackage" value="com.ossean.dao2" />
|
||||
<property name="basePackage" value="com.ossean.databaseDest" />
|
||||
<property name="sqlSessionFactory" ref="sqlSessionFactoryTwo"></property>
|
||||
</bean>
|
||||
</bean> -->
|
||||
|
||||
<tx:annotation-driven transaction-manager="transactionManager"/>
|
||||
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@
|
|||
</appender>
|
||||
|
||||
<appender name="file_log1" class="org.apache.log4j.DailyRollingFileAppender">
|
||||
<param name="File" value="./log/time_cost1.txt" />
|
||||
<param name="File" value="./log/time_cost.txt" />
|
||||
<param name="threshold" value="WARN" />
|
||||
<layout class="org.apache.log4j.PatternLayout">
|
||||
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
|
||||
|
|
|
|||
|
|
@ -1,172 +0,0 @@
|
|||
package com.ossean;
|
||||
|
||||
import info.debatty.java.stringsimilarity.*;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
|
||||
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
|
||||
|
||||
import com.ossean.dao.DBDest;
|
||||
import com.ossean.dao.DBSource;
|
||||
import com.ossean.dao.GatherDao;
|
||||
import com.ossean.dao2.PlatformProjectDao;
|
||||
import com.ossean.model.GatherProjectsModel;
|
||||
import com.ossean.model.SourceForgeProject;
|
||||
import com.ossean.util.MergeProjectNew2;
|
||||
import com.ossean.util.RegexHandler;
|
||||
import com.ossean.util.StringHandler;
|
||||
|
||||
@RunWith(SpringJUnit4ClassRunner.class)
|
||||
@ContextConfiguration({"classpath:/applicationContext.xml","classpath:/applicationContext-myBatis.xml"})
|
||||
public class MergeProjectNewTest extends AbstractJUnit4SpringContextTests {
|
||||
@Resource
|
||||
private DBSource dbSource;
|
||||
@Resource
|
||||
private GatherDao gatherDao;
|
||||
@Resource
|
||||
private PlatformProjectDao platformProjectDao;
|
||||
@Resource
|
||||
private DBDest dbDest;
|
||||
@Qualifier("mergeProjectNew2")
|
||||
@Autowired
|
||||
private MergeProjectNew2 mergeProjectNew2;
|
||||
@Qualifier("insertSynonyms")
|
||||
@Autowired
|
||||
private InsertSynonyms insertSynonyms;
|
||||
|
||||
|
||||
private static String pointerTableName = "edd_pointers_17";//记录去重进行到哪个地方了
|
||||
private static String sourceTableName = "gather_projects_test";//去重项目的来源，来自汇总程序生成的表
|
||||
private static String targetTableName = "edd_relations_17";//去重结果存储位置
|
||||
|
||||
@Test
|
||||
public void testGetSynonyms(){
|
||||
|
||||
GatherProjectsModel model = gatherDao.selectGPMById(sourceTableName, 959333);
|
||||
GatherProjectsModel model2 = gatherDao.selectGPMById(sourceTableName, 100014);
|
||||
List<String> sylist = insertSynonyms.getSynonyms(model);
|
||||
List<String> sylist2 = insertSynonyms.getSynonyms(model2);
|
||||
System.out.println(357644+":"+sylist);
|
||||
System.out.println(100014+":"+sylist2);
|
||||
|
||||
}
|
||||
|
||||
public static boolean contain2(List<String> inputList, String regex) {
|
||||
Pattern p = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
|
||||
for(int i=0;i<inputList.size();i++){
|
||||
Matcher m = p.matcher(inputList.get(i));
|
||||
boolean result = m.find();
|
||||
if(result==true){
|
||||
System.out.println(inputList.get(i));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUrl(){
|
||||
URL url;
|
||||
try {
|
||||
url = new URL("https://www.7-zip.org");//http://sourceforge.net/projects/gsevenzip
|
||||
String host = url.getHost();// 获取主机名
|
||||
System.out.println("host:"+host);// 结果 blog.csdn.net
|
||||
} catch (MalformedURLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSimilarity(){
|
||||
|
||||
String s1 = "Google Map Gps Cell Phone Tracker has a responsive design using twitter bootstrap";
|
||||
String s2 = "Track a GPS enabled cell phone on Google maps.";
|
||||
GatherProjectsModel model1 = gatherDao.selectGPMById("gather_projects", 515);
|
||||
GatherProjectsModel model2 = gatherDao.selectGPMById("gather_projects", 597699);
|
||||
String desc1 = StringHandler.getFirstSentence(model1.getDescription());
|
||||
String desc2 = StringHandler.getFirstSentence(model2.getDescription());
|
||||
NGram ngram = new NGram(4);
|
||||
System.out.println(desc1);
|
||||
System.out.println(desc2);
|
||||
System.out.println(1-ngram.distance(s1,s2));
|
||||
// JaroWinkler jw = new JaroWinkler();
|
||||
// System.out.println(jw.similarity(desc1, desc2));
|
||||
|
||||
// LongestCommonSubsequence lcs = new LongestCommonSubsequence();
|
||||
// // Will produce 4.0
|
||||
// System.out.println(lcs.distance(s1, s2));
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testComparePrj(){
|
||||
GatherProjectsModel model1 = gatherDao.selectGPMById("gather_projects", 1001009);
|
||||
GatherProjectsModel model2 = gatherDao.selectGPMById("gather_projects", 939809);
|
||||
//boolean isornot = mergeProjectNew.isTheSameAfterTFIDF(model1, model2,false);
|
||||
boolean isornot2 = mergeProjectNew2.isTheSame(model1, model2,true);
|
||||
NGram ngram = new NGram(4);
|
||||
System.out.println("similarity："+(1-ngram.distance(model1.getDescription(),model2.getDescription())));
|
||||
// /System.out.println(isornot);
|
||||
System.out.println(isornot2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHandleProject(){
|
||||
GatherProjectsModel model = gatherDao.selectGPMById("gather_projects_test",6026);//,100014,36707,357644,941234,
|
||||
mergeProjectNew2.handleNewProject(model);
|
||||
// List<GatherProjectsModel> list = new ArrayList<GatherProjectsModel>();
|
||||
// list.add(dbSource.getGatherProjectById("gather_projects_test", 202));
|
||||
// list.add(dbSource.getGatherProjectById("gather_projects_test", 515));
|
||||
// list.add(dbSource.getGatherProjectById("gather_projects_test", 742));
|
||||
// list.add(dbSource.getGatherProjectById("gather_projects_test", 767));
|
||||
// for(GatherProjectsModel model1:list){
|
||||
// mergeProjectNew2.handleNewProject(model1);
|
||||
// }
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLanguage(){
|
||||
String language = "C#, PHP, JavaScript, PL/SQL, Java, Objective-C 2.0";
|
||||
String aa="<Java>,<PHP>,<C#>";
|
||||
System.out.println(StringHandler.compareLanguage(aa,language));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testtest(){
|
||||
String [] aa = StringHandler.splitTagsByBracket("<Database><Information Analysis>");
|
||||
for(int i=0;i<aa.length;i++)
|
||||
System.out.println(aa[i]);
|
||||
System.out.println(RegexHandler.extractEngDecimalAndChinese("<java>"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPartDao(){
|
||||
List<GatherProjectsModel> list = new ArrayList<GatherProjectsModel>();
|
||||
list.add(gatherDao.selectGPMById("gather_projects_test", 175414));//sourceforge
|
||||
list.add(gatherDao.selectGPMById("gather_projects_test", 788279));
|
||||
list.add(gatherDao.selectGPMById("gather_projects_test", 45));//openhub
|
||||
list.add(gatherDao.selectGPMById("gather_projects_test", 426360));
|
||||
list.add(gatherDao.selectGPMById("gather_projects_test", 471974));
|
||||
list.add(gatherDao.selectGPMById("gather_projects_test", 770152));//oschina
|
||||
List<GatherProjectsModel> returnlist = mergeProjectNew2.returnDupModelList(gatherDao.selectGPMById("gather_projects_test", 175414),list,true);
|
||||
// GatherProjectsModel model = mergeProjectNew2.getUniqueSourceModel(list);
|
||||
// System.out.println(model.getName()+" "+model.getName());
|
||||
System.out.println(returnlist.size());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:context="http://www.springframework.org/schema/context"
|
||||
xmlns:aop="http://www.springframework.org/schema/aop"
|
||||
xmlns:p="http://www.springframework.org/schema/p"
|
||||
xmlns:tx="http://www.springframework.org/schema/tx"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans
|
||||
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
|
||||
http://www.springframework.org/schema/context
|
||||
http://www.springframework.org/schema/context/spring-context-3.0.xsd
|
||||
http://www.springframework.org/schema/tx
|
||||
http://www.springframework.org/schema/tx/spring-tx-3.0.xsd
|
||||
http://www.springframework.org/schema/aop
|
||||
http://www.springframework.org/schema/aop/spring-aop-3.0.xsd">
|
||||
|
||||
<!--SourceData -->
|
||||
<bean id="dataSourceOne" class="org.apache.commons.dbcp.BasicDataSource"
|
||||
destroy-method="close">
|
||||
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
|
||||
<property name="url"
|
||||
value="jdbc:mysql://localhost/ossean?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="root" />
|
||||
<property name="password" value="123456" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
<property name="testOnBorrow" value="true"/>
|
||||
</bean>
|
||||
<bean id="sqlSessionFactoryOne" class="org.mybatis.spring.SqlSessionFactoryBean">
|
||||
<property name="dataSource" ref="dataSourceOne" />
|
||||
</bean>
|
||||
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
|
||||
<property name="basePackage" value="com.ossean.dao" />
|
||||
<property name="sqlSessionFactory" ref="sqlSessionFactoryOne"></property>
|
||||
</bean>
|
||||
|
||||
|
||||
<bean id="dataSourceTwo" class="org.apache.commons.dbcp.BasicDataSource"
|
||||
destroy-method="close">
|
||||
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
|
||||
<property name="url"
|
||||
value="jdbc:mysql://localhost:3306/ossean?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="root" />
|
||||
<property name="password" value="123456" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
<property name="testOnBorrow" value="true"/>
|
||||
</bean>
|
||||
<bean id="sqlSessionFactoryTwo" class="org.mybatis.spring.SqlSessionFactoryBean">
|
||||
<property name="dataSource" ref="dataSourceTwo" />
|
||||
</bean>
|
||||
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
|
||||
<property name="basePackage" value="com.ossean.dao2" />
|
||||
<property name="sqlSessionFactory" ref="sqlSessionFactoryTwo"></property>
|
||||
</bean>
|
||||
|
||||
<tx:annotation-driven transaction-manager="transactionManager"/>
|
||||
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
|
||||
<property name="dataSource" ref="dataSourceOne" />
|
||||
</bean>
|
||||
|
||||
</beans>
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
||||
<properties>
|
||||
<comment>IK Analyzer 扩展配置</comment>
|
||||
<!--用户可以在这里配置自己的扩展字典 -->
|
||||
<entry key="ext_dict">ext.dic;</entry>
|
||||
|
||||
<!--用户可以在这里配置自己的扩展停止词字典-->
|
||||
<entry key="ext_stopwords">stopword.dic;</entry>
|
||||
|
||||
</properties>
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
|
||||
<properties>
|
||||
<comment>TableFlow</comment>
|
||||
<entry key="pointerTableName">edd_pointers</entry>
|
||||
<entry key="migrationTask">gather_tasks</entry>
|
||||
<entry key="sourceTableName">apache</entry>
|
||||
<entry key="targetTableName">gather_projects</entry>
|
||||
<entry key="sourceFields">id,null,project_name,project_descriprion,project_updated_time,project_created_time,null,Url,MD5(Url),null,languege,null,null,null,null,null,null,categories,null,history,"Apache",license,null</entry>
|
||||
<entry key="targetFields">id,deal_name,name,description,last_update_time,registered_time,followers_num,url,urlMD5,pageMD5,language,tags,contributors_num,monitor,status,download_num,visit_num,category,crawled_time,history,source,license,platform</entry>
|
||||
<entry key="waitDataTime">3600000</entry>
|
||||
<entry key="andWhere"> and id>=1</entry>
|
||||
<entry key="idsBegin">1</entry>
|
||||
<entry key="idsEnd">500000</entry>
|
||||
<entry key="idsIncrement">500</entry>
|
||||
</properties>
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:context="http://www.springframework.org/schema/context"
|
||||
xmlns:aop="http://www.springframework.org/schema/aop"
|
||||
xmlns:p="http://www.springframework.org/schema/p"
|
||||
xmlns:tx="http://www.springframework.org/schema/tx"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans
|
||||
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
|
||||
http://www.springframework.org/schema/context
|
||||
http://www.springframework.org/schema/context/spring-context-3.0.xsd
|
||||
http://www.springframework.org/schema/tx
|
||||
http://www.springframework.org/schema/tx/spring-tx-3.0.xsd
|
||||
http://www.springframework.org/schema/aop
|
||||
http://www.springframework.org/schema/aop/spring-aop-3.0.xsd">
|
||||
|
||||
<!--SourceData -->
|
||||
<bean id="dataSourceOne" class="org.apache.commons.dbcp.BasicDataSource"
|
||||
destroy-method="close">
|
||||
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
|
||||
<property name="url"
|
||||
value="jdbc:mysql://localhost/ossean?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="root" />
|
||||
<property name="password" value="123456" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
<property name="testOnBorrow" value="true"/>
|
||||
</bean>
|
||||
<bean id="sqlSessionFactoryOne" class="org.mybatis.spring.SqlSessionFactoryBean">
|
||||
<property name="dataSource" ref="dataSourceOne" />
|
||||
</bean>
|
||||
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
|
||||
<property name="basePackage" value="com.ossean.dao" />
|
||||
<property name="sqlSessionFactory" ref="sqlSessionFactoryOne"></property>
|
||||
</bean>
|
||||
|
||||
|
||||
<bean id="dataSourceTwo" class="org.apache.commons.dbcp.BasicDataSource"
|
||||
destroy-method="close">
|
||||
<property name="driverClassName" value="com.mysql.jdbc.Driver" />
|
||||
<property name="url"
|
||||
value="jdbc:mysql://localhost:3306/ossean?characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull&autoReconnect=true" />
|
||||
<property name="username" value="root" />
|
||||
<property name="password" value="123456" />
|
||||
<property name="validationQuery" value="SELECT 1" />
|
||||
<property name="testOnBorrow" value="true"/>
|
||||
</bean>
|
||||
<bean id="sqlSessionFactoryTwo" class="org.mybatis.spring.SqlSessionFactoryBean">
|
||||
<property name="dataSource" ref="dataSourceTwo" />
|
||||
</bean>
|
||||
<bean class="org.mybatis.spring.mapper.MapperScannerConfigurer">
|
||||
<property name="basePackage" value="com.ossean.dao2" />
|
||||
<property name="sqlSessionFactory" ref="sqlSessionFactoryTwo"></property>
|
||||
</bean>
|
||||
|
||||
<tx:annotation-driven transaction-manager="transactionManager"/>
|
||||
<bean id="transactionManager" class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
|
||||
<property name="dataSource" ref="dataSourceOne" />
|
||||
</bean>
|
||||
|
||||
</beans>
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:context="http://www.springframework.org/schema/context"
|
||||
xmlns:mvc="http://www.springframework.org/schema/mvc"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/mvc
|
||||
http://www.springframework.org/schema/mvc/spring-mvc-3.0.xsd
|
||||
http://www.springframework.org/schema/beans
|
||||
http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
|
||||
http://www.springframework.org/schema/context
|
||||
http://www.springframework.org/schema/context/spring-context-3.0.xsd">
|
||||
<context:annotation-config/>
|
||||
|
||||
<context:component-scan base-package="com.ossean"/>
|
||||
|
||||
</beans>
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,6 +0,0 @@
|
|||
sql server
|
||||
web server
|
||||
http server
|
||||
linux kernel
|
||||
face++
|
||||
一个
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue