update tags
This commit is contained in:
parent
6aca0e1139
commit
e5c17cb0ef
|
|
@ -12,18 +12,18 @@
|
|||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" path="src/main/resources"/>
|
||||
<classpathentry including="**/*.java" kind="src" path="src/main/resources"/>
|
||||
<classpathentry kind="src" path="src/main/assembly"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="lib" path="lib/IKAnalyzer-5.0.jar"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
||||
|
|
|
|||
|
|
@ -14,12 +14,12 @@
|
|||
</classpathentry>
|
||||
<classpathentry kind="src" path="src/main/assembly"/>
|
||||
<classpathentry kind="src" path="src/main/resource"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5">
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
package com.ossean.projectmanager;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.projectmanager.hotwords.UpdateOspTags;
|
||||
import com.ossean.projectmanager.projectsfilter.ProjectsFilter;
|
||||
|
||||
@Component
|
||||
public class UpdateOspTagsMain {
|
||||
@Qualifier("updateTags")
|
||||
@Autowired
|
||||
private UpdateOspTags updateTags;
|
||||
|
||||
public void start(){
|
||||
updateTags.updateOspTags();
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml");
|
||||
UpdateOspTagsMain mainClass = applicationContext.getBean(UpdateOspTagsMain.class);
|
||||
mainClass.start();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
package com.ossean.projectmanager.hotwords;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import javax.annotation.Resource;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import com.ossean.projectmanager.lasttabledao.OpenSourceProjectDao;
|
||||
import com.ossean.projectmanager.lasttabledao.TagDao;
|
||||
import com.ossean.projectmanager.lasttabledao.TaggingsDao;
|
||||
import com.ossean.projectmanager.model.OpenSourceProject;
|
||||
import com.ossean.projectmanager.model.Taggings;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author zhanyun
|
||||
*
|
||||
*/
|
||||
@Component("updateTags")
|
||||
public class UpdateOspTags {
|
||||
private Logger logger = LoggerFactory.getLogger(getClass());
|
||||
@Resource
|
||||
private OpenSourceProjectDao ospDao;
|
||||
@Resource
|
||||
private TagDao tagDao;
|
||||
@Resource
|
||||
private TaggingsDao taggingsDao;
|
||||
|
||||
/**
|
||||
* 根据taggings更新项目标签字段tags和权重更高的标签字段tags_for_search
|
||||
*/
|
||||
public void updateOspTags() {
|
||||
logger.info("start updateOspTags ......");
|
||||
int start = 0;
|
||||
int prjId = 0;
|
||||
while (start < 2000000) {
|
||||
List<OpenSourceProject> ospList = ospDao.getProjectsByBatch(start,
|
||||
5000);
|
||||
for (OpenSourceProject osp : ospList) {
|
||||
prjId = osp.getId();
|
||||
List<Taggings> taggingsList = taggingsDao.getByOspId(prjId);
|
||||
String tempTag = "";
|
||||
String tempTagForSearch = "";
|
||||
for (Taggings taggings : taggingsList) {
|
||||
String tag = tagDao.getNameById(taggings.getTag_id());
|
||||
tempTag += "<" + tag + ">,";
|
||||
if(taggings.getDisagree_num() > 50){
|
||||
tempTagForSearch += "<" + tag + ">,";
|
||||
}
|
||||
}
|
||||
String curPrjTag = "";
|
||||
String curPrjTagForSearch = "";
|
||||
if (tempTag.length() > 0) {
|
||||
curPrjTag = tempTag.substring(0, tempTag.length() - 1);
|
||||
}
|
||||
if (tempTagForSearch.length() > 0) {
|
||||
curPrjTagForSearch = tempTagForSearch.substring(0, tempTagForSearch.length() - 1);
|
||||
}
|
||||
ospDao.updatePrjTags(prjId, curPrjTag, curPrjTagForSearch);
|
||||
logger.info("currentPrjId: " + prjId);
|
||||
}
|
||||
logger.info("last prj batch end, currentPrjId: " + prjId);
|
||||
start = prjId + 5000;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -15,17 +15,27 @@ public interface OpenSourceProjectDao {
|
|||
public List<OpenSourceProject> getProjectsByBatch(
|
||||
@Param("start") int start, @Param("size") int size);
|
||||
|
||||
//更新项目标签字段tags和权重更高的标签字段tags_for_search
|
||||
@Update("update open_source_projects set tags=#{tags}, tags_for_search = #{tagsForSearch} where id=#{id}")
|
||||
public void updatePrjTags(@Param("id") int id,
|
||||
@Param("tags") String tags,
|
||||
@Param("tagsForSearch") String tagsForSearch);
|
||||
|
||||
// 对项目标签属性进行更新
|
||||
@Update("update open_source_projects set tags=#{tags} where id=#{id}")
|
||||
@Update("update open_source_projects set tags=#{tags}, tags_for_search = #{tagsForSearch} where id=#{id}")
|
||||
public void updateTagsOfProject(@Param("id") int id,
|
||||
@Param("tags") String tags);
|
||||
|
||||
// 批量获取项目
|
||||
@Select("select id,source,url,filtration from open_source_projects limit #{batchSize}")
|
||||
public List<OpenSourceProject> getBatchPrjs(@Param("batchSize") int batchSize);
|
||||
public List<OpenSourceProject> getBatchPrjs(
|
||||
@Param("batchSize") int batchSize);
|
||||
|
||||
// filtration为1表示保留,为2表示之前保留的且已处理,为0表示不保留
|
||||
@Update("update open_source_projects set filtration = #{filtration} where id = #{prjId}")
|
||||
public void updateFiltratedPrj(@Param("prjId") int prjId,
|
||||
@Param("filtration") int filtration);
|
||||
|
||||
@Update("UPDATE open_source_projects SET tags = ''")
|
||||
public void emptyOspTags();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,5 +16,9 @@ public interface TagDao {
|
|||
//向tags表插入数据
|
||||
@Insert("insert into tags (name) values (#{name})")
|
||||
public void insertTag(@Param("name") String name);
|
||||
|
||||
|
||||
//根据标签id获得标签名
|
||||
@Select("SELECT name FROM tags WHERE id = #{tagId}")
|
||||
public String getNameById(@Param("tagId") int tagId);
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,27 +10,33 @@ import org.apache.ibatis.annotations.Update;
|
|||
import com.ossean.projectmanager.model.Taggings;
|
||||
|
||||
public interface TaggingsDao {
|
||||
|
||||
//根据帖子id读取taggings表中与帖子相关联的taggings记录
|
||||
|
||||
// 根据帖子id读取taggings表中与帖子相关联的taggings记录
|
||||
@Select("select * from taggings where taggable_id=#{memoId} and taggable_type='RelativeMemo'")
|
||||
public List<Taggings> getByMemoId(@Param("memoId") int memoId);
|
||||
|
||||
|
||||
//向taggings表中插入由匹配帖子标签得到的新项目标签
|
||||
|
||||
// 根据项目id获得taggings
|
||||
@Select("SELECT * FROM taggings WHERE taggable_id = #{ospId} AND taggable_type = 'OpenSourceProject'")
|
||||
public List<Taggings> getByOspId(@Param("ospId") int ospId);
|
||||
|
||||
// 向taggings表中插入由匹配帖子标签得到的新项目标签
|
||||
@Insert("insert into taggings (tag_id,taggable_id,taggable_type,disagree_num,context,created_at,tag_source) values (#{item.tag_id},#{item.taggable_id},#{item.taggable_type},#{item.disagree_num},#{item.context},now(),#{item.tag_source})")
|
||||
public void insertTaggings(@Param("item") Taggings item);
|
||||
|
||||
|
||||
//查看taggings表中是否存在要查询的标签
|
||||
|
||||
// 查看taggings表中是否存在要查询的标签
|
||||
@Select("select * from taggings where tag_id=#{item.tag_id} and taggable_type=#{item.taggable_type} and taggable_id=#{item.taggable_id}")
|
||||
public List<Taggings> findTaggings(@Param("item") Taggings item);
|
||||
|
||||
//修改disagree_num值
|
||||
@Update("update taggings set disagree_num=#{value} where taggable_id=#{taggable_id} AND taggable_type=#{taggable_type} AND tag_id=#{tag_id} ")
|
||||
public void updateDisagreeNum(@Param("value") int value, @Param("taggable_id") int taggable_id, @Param("taggable_type") String taggable_type, @Param("tag_id") int tag_id);
|
||||
|
||||
|
||||
//获取disagree_num值
|
||||
// 修改disagree_num值
|
||||
@Update("update taggings set disagree_num=#{value} where taggable_id=#{taggable_id} AND taggable_type=#{taggable_type} AND tag_id=#{tag_id} ")
|
||||
public void updateDisagreeNum(@Param("value") int value,
|
||||
@Param("taggable_id") int taggable_id,
|
||||
@Param("taggable_type") String taggable_type,
|
||||
@Param("tag_id") int tag_id);
|
||||
|
||||
// 获取disagree_num值
|
||||
@Select("select disagree_num from taggings where tag_id=#{tag_id} and taggable_id=#{taggable_id} and taggable_type=#{taggable_type}")
|
||||
public int getDisagreeNum(@Param("tag_id") int tag_id, @Param("taggable_id") int taggable_id, @Param("taggable_type") String taggable_type);
|
||||
public int getDisagreeNum(@Param("tag_id") int tag_id,
|
||||
@Param("taggable_id") int taggable_id,
|
||||
@Param("taggable_type") String taggable_type);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,28 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" output="target/classes" path="src/main/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
|
||||
<attributes>
|
||||
<attribute name="optional" value="true"/>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="src" path="src/main/resource"/>
|
||||
<classpathentry kind="src" path="src/main/assembly"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
|
||||
<attributes>
|
||||
<attribute name="maven.pomderived" value="true"/>
|
||||
</attributes>
|
||||
</classpathentry>
|
||||
<classpathentry kind="output" path="target/classes"/>
|
||||
</classpath>
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
find ./target/classes -name "*.properties"|xargs rm -f
|
||||
find ./target/classes -name "*.xml"|xargs rm -f
|
||||
find ./target/classes -name "*.dic"|xargs rm -f
|
||||
|
||||
#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib
|
||||
|
||||
tmp='./bin/resources'
|
||||
tmp='./target/classes':$tmp
|
||||
tmp='./target/org.ossean.updatetags-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp
|
||||
|
||||
CLASSPATH=$tmp:$CLASSPATH
|
||||
|
||||
|
||||
echo $CLASSPATH
|
||||
JVM_ARGS="-Xmn48m -Xmx200m -Xms200m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2"
|
||||
#echo JVM_ARGS=$JVM_ARGS
|
||||
#ulimit -n 400000
|
||||
#echo "" > nohup.out
|
||||
java $JVM_ARGS -classpath $CLASSPATH OSSEAN.org.ossean.updatetags.Main>>info.log 2>&1 &
|
||||
Binary file not shown.
|
|
@ -1,58 +0,0 @@
|
|||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>OSSEAN</groupId>
|
||||
<artifactId>org.ossean.updatetags</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<build>
|
||||
<sourceDirectory>${basedir}/src</sourceDirectory>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>${basedir}/src/main/resources</directory>
|
||||
<excludes>
|
||||
<exclude>*.xml</exclude>
|
||||
<exclude>*.properties</exclude>
|
||||
</excludes>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>1.7</source>
|
||||
<target>1.7</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.5.1</version>
|
||||
<configuration>
|
||||
<descriptors>
|
||||
<descriptor>src/main/assembly/assembly.xml</descriptor>
|
||||
</descriptors>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
|
||||
</build>
|
||||
|
||||
<dependencies>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
<version>5.1.30</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<version>1.7.7</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
|
|
@ -1 +0,0 @@
|
|||
0 0
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
<assembly
|
||||
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
|
||||
<!-- TODO: a jarjar format would be better -->
|
||||
<id>jar-with-dependencies-without-resources</id>
|
||||
<formats>
|
||||
<format>dir</format>
|
||||
</formats>
|
||||
<includeBaseDirectory>false</includeBaseDirectory>
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<useProjectArtifact>false</useProjectArtifact>
|
||||
<unpack>false</unpack>
|
||||
<scope>runtime</scope>
|
||||
</dependencySet>
|
||||
<dependencySet>
|
||||
<outputDirectory>/</outputDirectory>
|
||||
<useProjectArtifact>false</useProjectArtifact>
|
||||
<unpack>false</unpack>
|
||||
<scope>system</scope>
|
||||
</dependencySet>
|
||||
</dependencySets>
|
||||
</assembly>
|
||||
|
|
@ -1,255 +0,0 @@
|
|||
package OSSEAN.org.ossean.updatetags;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Scanner;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author zhanyun
|
||||
*
|
||||
*/
|
||||
public class GetData {
|
||||
private Logger logger = LoggerFactory.getLogger(getClass());
|
||||
private static File record;
|
||||
private int prjHistory; // the record of project after updating last time
|
||||
private int memoHistory; // the record of memo after matching last time
|
||||
|
||||
public int getPrjHistory() {
|
||||
return prjHistory;
|
||||
}
|
||||
|
||||
public void setPrjHistory(int prjHistory) {
|
||||
this.prjHistory = prjHistory;
|
||||
}
|
||||
|
||||
public int getMemoHistory() {
|
||||
return memoHistory;
|
||||
}
|
||||
|
||||
public void setMemoHistory(int memoHistory) {
|
||||
this.memoHistory = memoHistory;
|
||||
}
|
||||
|
||||
public GetData(){
|
||||
record = new File("record.txt");
|
||||
this.getMatchRecord();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public Connection getConnection() {
|
||||
Connection con = null;
|
||||
if (con == null) {
|
||||
try {
|
||||
Class.forName("com.mysql.jdbc.Driver");
|
||||
con = DriverManager.getConnection(
|
||||
"jdbc:mysql://192.168.80.130:3306/ossean_production?useUnicode=true&characterEncoding=utf-8"
|
||||
, "trustie", "1234");// 鍒涘缓鏁版嵁杩炴帴
|
||||
|
||||
} catch (Exception e) {
|
||||
logger.info("连接错误" + e.getMessage());
|
||||
}
|
||||
return con;
|
||||
}
|
||||
return con;
|
||||
}
|
||||
|
||||
public void getMatchRecord(){
|
||||
try {
|
||||
Scanner in = new Scanner(record);
|
||||
this.setPrjHistory(in.nextInt());
|
||||
this.setMemoHistory(in.nextInt());
|
||||
in.close();
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.info(e.getStackTrace().toString());
|
||||
}
|
||||
}
|
||||
|
||||
public void writeIntoRecord(int prjId, int memoId){
|
||||
try {
|
||||
FileOutputStream fout = new FileOutputStream(record);
|
||||
fout.write((prjId + "\t" + memoId).getBytes());
|
||||
fout.close();
|
||||
} catch (FileNotFoundException e) {
|
||||
logger.info(e.getStackTrace().toString());
|
||||
} catch (IOException e) {
|
||||
logger.info(e.getStackTrace().toString());
|
||||
}
|
||||
}
|
||||
|
||||
public void updateOspTags(Connection conn) {
|
||||
logger.info("start updateOspTags ......");
|
||||
String ospSql = "SELECT id FROM open_source_projects WHERE id > ? AND id <= ?";
|
||||
String sql = "SELECT tags.`name` FROM taggings, tags WHERE taggings.taggable_id = ? AND taggings.taggable_type = 'OpenSourceProject' AND tags.id = taggings.tag_id";
|
||||
String deleteOspTagsSql = "UPDATE open_source_projects SET tags = '';";
|
||||
String updateOspSql = "UPDATE open_source_projects SET tags = ? WHERE id = ?;";
|
||||
int prjId = this.getPrjHistory();
|
||||
int end = prjId + 5000;
|
||||
try {
|
||||
PreparedStatement ps3 = conn.prepareStatement(deleteOspTagsSql);
|
||||
ps3.executeUpdate();
|
||||
ps3.close();
|
||||
logger.info("已清空tags...");
|
||||
while (end < 20000) {
|
||||
PreparedStatement ps2 = conn.prepareStatement(ospSql);
|
||||
ps2.setInt(1, prjId);
|
||||
ps2.setInt(2, end);
|
||||
ResultSet pIds = ps2.executeQuery();
|
||||
if (!pIds.next()) {
|
||||
prjId = end;
|
||||
} else {
|
||||
pIds.beforeFirst();
|
||||
}
|
||||
while (pIds.next()) {
|
||||
prjId = pIds.getInt(1);
|
||||
PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ps.setInt(1, prjId);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
String temp_tag = "";
|
||||
while (rs.next()) {
|
||||
temp_tag += "<" + rs.getString(1) + ">,";
|
||||
}
|
||||
String curPrjTag = "";
|
||||
if (temp_tag.length() > 0) {
|
||||
curPrjTag = temp_tag
|
||||
.substring(0, temp_tag.length() - 1);
|
||||
}
|
||||
PreparedStatement ps1 = conn.prepareStatement(updateOspSql);
|
||||
ps1.setString(1, curPrjTag);
|
||||
ps1.setInt(2, prjId);
|
||||
ps1.executeUpdate();
|
||||
conn.commit();
|
||||
rs.close();
|
||||
ps.close();
|
||||
ps1.close();
|
||||
logger.info("currentPrjId: " + prjId);
|
||||
}
|
||||
logger.info("last prj batch end, currentPrjId: " + prjId);
|
||||
end = prjId + 5000;
|
||||
}
|
||||
|
||||
} catch (SQLException e) {
|
||||
logger.info(e.getStackTrace().toString());
|
||||
}
|
||||
}
|
||||
|
||||
public void updateOspTagsForSearch(Connection conn) {
|
||||
logger.info("start updateOspTagsForSearch ......");
|
||||
String ospSql = "SELECT id FROM open_source_projects WHERE id > ? AND id <= ?";
|
||||
String sql = "SELECT tags.`name` FROM taggings, tags WHERE taggings.taggable_id = ? AND taggings.taggable_type = 'OpenSourceProject' AND tags.id = taggings.tag_id AND taggings.disagree_num > 50";
|
||||
String deleteOspTagsSql = "UPDATE open_source_projects SET tags_for_search = '';";
|
||||
String updateOspSql = "UPDATE open_source_projects SET tags_for_search = ? WHERE id = ?;";
|
||||
int prjId = this.getPrjHistory();
|
||||
int end = prjId + 5000;
|
||||
try {
|
||||
PreparedStatement ps3 = conn.prepareStatement(deleteOspTagsSql);
|
||||
ps3.executeUpdate();
|
||||
ps3.close();
|
||||
logger.info("已清空tags_for_search...");
|
||||
while (end < 400000) {
|
||||
PreparedStatement ps2 = conn.prepareStatement(ospSql);
|
||||
ps2.setInt(1, prjId);
|
||||
ps2.setInt(2, end);
|
||||
ResultSet pIds = ps2.executeQuery();
|
||||
if (!pIds.next()) {
|
||||
prjId = end;
|
||||
} else {
|
||||
pIds.beforeFirst();
|
||||
}
|
||||
while (pIds.next()) {
|
||||
prjId = pIds.getInt(1);
|
||||
PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ps.setInt(1, prjId);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
String temp_tag = "";
|
||||
while (rs.next()) {
|
||||
temp_tag += "<" + rs.getString(1) + ">,";
|
||||
}
|
||||
String curPrjTag = "";
|
||||
if (temp_tag.length() > 0) {
|
||||
curPrjTag = temp_tag
|
||||
.substring(0, temp_tag.length() - 1);
|
||||
}
|
||||
PreparedStatement ps1 = conn.prepareStatement(updateOspSql);
|
||||
ps1.setString(1, curPrjTag);
|
||||
ps1.setInt(2, prjId);
|
||||
ps1.executeUpdate();
|
||||
conn.commit();
|
||||
rs.close();
|
||||
ps.close();
|
||||
ps1.close();
|
||||
logger.info("currentPrjId: " + prjId);
|
||||
}
|
||||
logger.info("last prj batch end, currentPrjId: " + prjId);
|
||||
end = prjId + 5000;
|
||||
}
|
||||
|
||||
} catch (SQLException e) {
|
||||
logger.info(e.getStackTrace().toString());
|
||||
}
|
||||
}
|
||||
|
||||
public void updateMemoTags(Connection conn) {
|
||||
logger.info("start updateMemoTags ......");
|
||||
String memoSql = "SELECT id FROM relative_memos WHERE id > ? AND id <= ?";
|
||||
String sql = "SELECT tags.`name` FROM taggings, tags WHERE taggings.taggable_id = ? AND taggings.taggable_type = 'RelativeMemo' AND tags.id = taggings.tag_id";
|
||||
String updateMemoSql = "UPDATE relative_memos SET tags = ? WHERE id = ?";
|
||||
int memoId = this.getMemoHistory();
|
||||
int end = memoId + 5000;
|
||||
try {
|
||||
while (end < 4000000) {
|
||||
PreparedStatement ps2 = conn.prepareStatement(memoSql);
|
||||
ps2.setInt(1, memoId);
|
||||
ps2.setInt(2, end);
|
||||
ResultSet memoIds = ps2.executeQuery();
|
||||
if (!memoIds.next()) {
|
||||
memoId = end;
|
||||
} else {
|
||||
memoIds.beforeFirst();
|
||||
}
|
||||
|
||||
while (memoIds.next()) {
|
||||
memoId = memoIds.getInt(1);
|
||||
PreparedStatement ps = conn.prepareStatement(sql);
|
||||
ps.setInt(1, memoId);
|
||||
ResultSet rs = ps.executeQuery();
|
||||
String temp_tag = "";
|
||||
while (rs.next()) {
|
||||
temp_tag += "<" + rs.getString(1) + ">,";
|
||||
}
|
||||
String curMemoTag = "";
|
||||
if (temp_tag.length() > 0) {
|
||||
curMemoTag = temp_tag.substring(0,
|
||||
temp_tag.length() - 1);
|
||||
}
|
||||
PreparedStatement ps1 = conn
|
||||
.prepareStatement(updateMemoSql);
|
||||
ps1.setString(1, curMemoTag);
|
||||
ps1.setInt(2, memoId);
|
||||
ps1.executeUpdate();
|
||||
conn.commit();
|
||||
rs.close();
|
||||
ps.close();
|
||||
ps1.close();
|
||||
logger.info("currentMemoId: " + memoId);
|
||||
}
|
||||
logger.info("last memo batch end, currentMemoId: " + memoId);
|
||||
end = memoId + 5000;
|
||||
}
|
||||
|
||||
} catch (SQLException e) {
|
||||
logger.info(e.getStackTrace().toString());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
package OSSEAN.org.ossean.updatetags;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import com.mysql.jdbc.Connection;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author zhanyun
|
||||
*
|
||||
*/
|
||||
public class Main {
|
||||
|
||||
public static void main(String[] args) throws SQLException {
|
||||
GetData getData = new GetData();
|
||||
Connection conn;
|
||||
conn = (Connection) getData.getConnection();
|
||||
conn.setAutoCommit(false);
|
||||
getData.updateOspTagsForSearch(conn);
|
||||
getData.updateOspTags(conn);
|
||||
conn.close();
|
||||
Connection conn1;
|
||||
conn1 = (Connection) getData.getConnection();
|
||||
conn1.setAutoCommit(false);
|
||||
getData.updateMemoTags(conn1);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
|
||||
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
|
||||
|
||||
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
|
||||
<layout class="org.apache.log4j.PatternLayout">
|
||||
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
|
||||
</layout>
|
||||
</appender>
|
||||
|
||||
<appender name="MAIL" class="org.apache.log4j.net.SMTPAppender">
|
||||
<param name="threshold" value="info" />
|
||||
<param name="BufferSize" value="512" />
|
||||
<param name="From" value="gcm365111@126.com" />
|
||||
<param name="SMTPHost" value="SMTP.126.com" />
|
||||
<param name="Subject" value="this is test" />
|
||||
<param name="SMTPUsername" value="gcm365111@126.com" />
|
||||
<param name="SMTPPassword" value="03023651gcm" />
|
||||
<param name="to" value="gcm3651@126.com" />
|
||||
<layout class="org.apache.log4j.PatternLayout">
|
||||
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
|
||||
</layout>
|
||||
</appender>
|
||||
|
||||
<appender name="file" class="org.apache.log4j.DailyRollingFileAppender">
|
||||
<param name="File" value="./log/info.log" />
|
||||
<layout class="org.apache.log4j.PatternLayout">
|
||||
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
|
||||
</layout>
|
||||
</appender>
|
||||
|
||||
|
||||
|
||||
<root>
|
||||
<level value="info" />
|
||||
<appender-ref ref="file" />
|
||||
<appender-ref ref="stdout" />
|
||||
</root>
|
||||
|
||||
</log4j:configuration>
|
||||
Loading…
Reference in New Issue