diff --git a/ow2_match/.classpath b/ow2_match/.classpath index 78bd4b41f..79922f78f 100644 --- a/ow2_match/.classpath +++ b/ow2_match/.classpath @@ -12,18 +12,18 @@ - + - - - - - + + + + + diff --git a/project_manager/.classpath b/project_manager/.classpath index 970d43f69..1cef3054c 100644 --- a/project_manager/.classpath +++ b/project_manager/.classpath @@ -14,12 +14,12 @@ - + - + diff --git a/project_manager/src/main/java/com/ossean/projectmanager/UpdateOspTagsMain.java b/project_manager/src/main/java/com/ossean/projectmanager/UpdateOspTagsMain.java new file mode 100644 index 000000000..9eadf04ab --- /dev/null +++ b/project_manager/src/main/java/com/ossean/projectmanager/UpdateOspTagsMain.java @@ -0,0 +1,27 @@ +package com.ossean.projectmanager; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.stereotype.Component; + +import com.ossean.projectmanager.hotwords.UpdateOspTags; +import com.ossean.projectmanager.projectsfilter.ProjectsFilter; + +@Component +public class UpdateOspTagsMain { + @Qualifier("updateTags") + @Autowired + private UpdateOspTags updateTags; + + public void start(){ + updateTags.updateOspTags(); + } + + public static void main(String[] args){ + ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:/applicationContext*.xml"); + UpdateOspTagsMain mainClass = applicationContext.getBean(UpdateOspTagsMain.class); + mainClass.start(); + } +} diff --git a/project_manager/src/main/java/com/ossean/projectmanager/hotwords/UpdateOspTags.java b/project_manager/src/main/java/com/ossean/projectmanager/hotwords/UpdateOspTags.java new file mode 100644 index 000000000..838242103 --- /dev/null +++ b/project_manager/src/main/java/com/ossean/projectmanager/hotwords/UpdateOspTags.java @@ -0,0 +1,71 @@ +package com.ossean.projectmanager.hotwords; + +import java.util.List; + +import javax.annotation.Resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.ossean.projectmanager.lasttabledao.OpenSourceProjectDao; +import com.ossean.projectmanager.lasttabledao.TagDao; +import com.ossean.projectmanager.lasttabledao.TaggingsDao; +import com.ossean.projectmanager.model.OpenSourceProject; +import com.ossean.projectmanager.model.Taggings; + +/** + * + * @author zhanyun + * + */ +@Component("updateTags") +public class UpdateOspTags { + private Logger logger = LoggerFactory.getLogger(getClass()); + @Resource + private OpenSourceProjectDao ospDao; + @Resource + private TagDao tagDao; + @Resource + private TaggingsDao taggingsDao; + + /** + * 根据taggings更新项目标签字段tags和权重更高的标签字段tags_for_search + */ + public void updateOspTags() { + logger.info("start updateOspTags ......"); + int start = 0; + int prjId = 0; + while (start < 2000000) { + List ospList = ospDao.getProjectsByBatch(start, + 5000); + for (OpenSourceProject osp : ospList) { + prjId = osp.getId(); + List taggingsList = taggingsDao.getByOspId(prjId); + String tempTag = ""; + String tempTagForSearch = ""; + for (Taggings taggings : taggingsList) { + String tag = tagDao.getNameById(taggings.getTag_id()); + tempTag += "<" + tag + ">,"; + if(taggings.getDisagree_num() > 50){ + tempTagForSearch += "<" + tag + ">,"; + } + } + String curPrjTag = ""; + String curPrjTagForSearch = ""; + if (tempTag.length() > 0) { + curPrjTag = tempTag.substring(0, tempTag.length() - 1); + } + if (tempTagForSearch.length() > 0) { + curPrjTagForSearch = tempTagForSearch.substring(0, tempTagForSearch.length() - 1); + } + ospDao.updatePrjTags(prjId, curPrjTag, curPrjTagForSearch); + logger.info("currentPrjId: " + prjId); + } + logger.info("last prj batch end, currentPrjId: " + prjId); + start = prjId + 5000; + } + + } + +} diff --git a/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/OpenSourceProjectDao.java b/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/OpenSourceProjectDao.java index e05991ba9..67f55567b 100644 --- a/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/OpenSourceProjectDao.java +++ b/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/OpenSourceProjectDao.java @@ -15,17 +15,27 @@ public interface OpenSourceProjectDao { public List getProjectsByBatch( @Param("start") int start, @Param("size") int size); + //更新项目标签字段tags和权重更高的标签字段tags_for_search + @Update("update open_source_projects set tags=#{tags}, tags_for_search = #{tagsForSearch} where id=#{id}") + public void updatePrjTags(@Param("id") int id, + @Param("tags") String tags, + @Param("tagsForSearch") String tagsForSearch); + // 对项目标签属性进行更新 - @Update("update open_source_projects set tags=#{tags} where id=#{id}") + @Update("update open_source_projects set tags=#{tags}, tags_for_search = #{tagsForSearch} where id=#{id}") public void updateTagsOfProject(@Param("id") int id, @Param("tags") String tags); // 批量获取项目 @Select("select id,source,url,filtration from open_source_projects limit #{batchSize}") - public List getBatchPrjs(@Param("batchSize") int batchSize); + public List getBatchPrjs( + @Param("batchSize") int batchSize); // filtration为1表示保留,为2表示之前保留的且已处理,为0表示不保留 @Update("update open_source_projects set filtration = #{filtration} where id = #{prjId}") public void updateFiltratedPrj(@Param("prjId") int prjId, @Param("filtration") int filtration); + + @Update("UPDATE open_source_projects SET tags = ''") + public void emptyOspTags(); } diff --git a/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/TagDao.java b/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/TagDao.java index 70aa129ce..d5c23066f 100644 --- a/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/TagDao.java +++ b/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/TagDao.java @@ -16,5 +16,9 @@ public interface TagDao { //向tags表插入数据 @Insert("insert into tags (name) values (#{name})") public void insertTag(@Param("name") String name); - + + //根据标签id获得标签名 + @Select("SELECT name FROM tags WHERE id = #{tagId}") + public String getNameById(@Param("tagId") int tagId); + } diff --git a/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/TaggingsDao.java b/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/TaggingsDao.java index 9bca8b31c..419f63d59 100644 --- a/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/TaggingsDao.java +++ b/project_manager/src/main/java/com/ossean/projectmanager/lasttabledao/TaggingsDao.java @@ -10,27 +10,33 @@ import org.apache.ibatis.annotations.Update; import com.ossean.projectmanager.model.Taggings; public interface TaggingsDao { - - //根据帖子id读取taggings表中与帖子相关联的taggings记录 + + // 根据帖子id读取taggings表中与帖子相关联的taggings记录 @Select("select * from taggings where taggable_id=#{memoId} and taggable_type='RelativeMemo'") public List getByMemoId(@Param("memoId") int memoId); - - - //向taggings表中插入由匹配帖子标签得到的新项目标签 + + // 根据项目id获得taggings + @Select("SELECT * FROM taggings WHERE taggable_id = #{ospId} AND taggable_type = 'OpenSourceProject'") + public List getByOspId(@Param("ospId") int ospId); + + // 向taggings表中插入由匹配帖子标签得到的新项目标签 @Insert("insert into taggings (tag_id,taggable_id,taggable_type,disagree_num,context,created_at,tag_source) values (#{item.tag_id},#{item.taggable_id},#{item.taggable_type},#{item.disagree_num},#{item.context},now(),#{item.tag_source})") public void insertTaggings(@Param("item") Taggings item); - - - //查看taggings表中是否存在要查询的标签 + + // 查看taggings表中是否存在要查询的标签 @Select("select * from taggings where tag_id=#{item.tag_id} and taggable_type=#{item.taggable_type} and taggable_id=#{item.taggable_id}") public List findTaggings(@Param("item") Taggings item); - - //修改disagree_num值 - @Update("update taggings set disagree_num=#{value} where taggable_id=#{taggable_id} AND taggable_type=#{taggable_type} AND tag_id=#{tag_id} ") - public void updateDisagreeNum(@Param("value") int value, @Param("taggable_id") int taggable_id, @Param("taggable_type") String taggable_type, @Param("tag_id") int tag_id); - - //获取disagree_num值 + // 修改disagree_num值 + @Update("update taggings set disagree_num=#{value} where taggable_id=#{taggable_id} AND taggable_type=#{taggable_type} AND tag_id=#{tag_id} ") + public void updateDisagreeNum(@Param("value") int value, + @Param("taggable_id") int taggable_id, + @Param("taggable_type") String taggable_type, + @Param("tag_id") int tag_id); + + // 获取disagree_num值 @Select("select disagree_num from taggings where tag_id=#{tag_id} and taggable_id=#{taggable_id} and taggable_type=#{taggable_type}") - public int getDisagreeNum(@Param("tag_id") int tag_id, @Param("taggable_id") int taggable_id, @Param("taggable_type") String taggable_type); + public int getDisagreeNum(@Param("tag_id") int tag_id, + @Param("taggable_id") int taggable_id, + @Param("taggable_type") String taggable_type); } diff --git a/update_tags/.classpath b/update_tags/.classpath deleted file mode 100644 index 4e401aeb4..000000000 --- a/update_tags/.classpath +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/update_tags/bin/start.sh b/update_tags/bin/start.sh deleted file mode 100644 index 48de7da9e..000000000 --- a/update_tags/bin/start.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -find ./target/classes -name "*.properties"|xargs rm -f -find ./target/classes -name "*.xml"|xargs rm -f -find ./target/classes -name "*.dic"|xargs rm -f - -#export CLASSPATH=$CURR_DIR/lib:$CURR_DIR:$JAVA_HOME/lib:$JAVA_HOME/jre/lib - -tmp='./bin/resources' -tmp='./target/classes':$tmp -tmp='./target/org.ossean.updatetags-0.0.1-SNAPSHOT-jar-with-dependencies-without-resources/*':$tmp - -CLASSPATH=$tmp:$CLASSPATH - - -echo $CLASSPATH -JVM_ARGS="-Xmn48m -Xmx200m -Xms200m -XX:NewRatio=4 -XX:SurvivorRatio=4 -XX:MaxTenuringThreshold=2" -#echo JVM_ARGS=$JVM_ARGS -#ulimit -n 400000 -#echo "" > nohup.out -java $JVM_ARGS -classpath $CLASSPATH OSSEAN.org.ossean.updatetags.Main>>info.log 2>&1 & \ No newline at end of file diff --git a/update_tags/lib/mysql-connector-java-5.1.29-bin.jar b/update_tags/lib/mysql-connector-java-5.1.29-bin.jar deleted file mode 100644 index c683dfeeb..000000000 Binary files a/update_tags/lib/mysql-connector-java-5.1.29-bin.jar and /dev/null differ diff --git a/update_tags/pom.xml b/update_tags/pom.xml deleted file mode 100644 index 02751924f..000000000 --- a/update_tags/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - 4.0.0 - - OSSEAN - org.ossean.updatetags - 0.0.1-SNAPSHOT - - ${basedir}/src - - - ${basedir}/src/main/resources - - *.xml - *.properties - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.7 - 1.7 - - - - maven-assembly-plugin - 2.5.1 - - - src/main/assembly/assembly.xml - - - - - - - - - - - - mysql - mysql-connector-java - 5.1.30 - - - - org.slf4j - slf4j-log4j12 - 1.7.7 - - - - - \ No newline at end of file diff --git a/update_tags/record.txt b/update_tags/record.txt deleted file mode 100644 index 4a3b60a28..000000000 --- a/update_tags/record.txt +++ /dev/null @@ -1 +0,0 @@ -0 0 \ No newline at end of file diff --git a/update_tags/src/main/assembly/assembly.xml b/update_tags/src/main/assembly/assembly.xml deleted file mode 100644 index baf8ba912..000000000 --- a/update_tags/src/main/assembly/assembly.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - jar-with-dependencies-without-resources - - dir - - false - - - / - false - false - runtime - - - / - false - false - system - - - \ No newline at end of file diff --git a/update_tags/src/main/java/OSSEAN/org/ossean/updatetags/GetData.java b/update_tags/src/main/java/OSSEAN/org/ossean/updatetags/GetData.java deleted file mode 100644 index d76b901c6..000000000 --- a/update_tags/src/main/java/OSSEAN/org/ossean/updatetags/GetData.java +++ /dev/null @@ -1,255 +0,0 @@ -package OSSEAN.org.ossean.updatetags; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Scanner; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * - * @author zhanyun - * - */ -public class GetData { - private Logger logger = LoggerFactory.getLogger(getClass()); - private static File record; - private int prjHistory; // the record of project after updating last time - private int memoHistory; // the record of memo after matching last time - - public int getPrjHistory() { - return prjHistory; - } - - public void setPrjHistory(int prjHistory) { - this.prjHistory = prjHistory; - } - - public int getMemoHistory() { - return memoHistory; - } - - public void setMemoHistory(int memoHistory) { - this.memoHistory = memoHistory; - } - - public GetData(){ - record = new File("record.txt"); - this.getMatchRecord(); - } - - @SuppressWarnings("unused") - public Connection getConnection() { - Connection con = null; - if (con == null) { - try { - Class.forName("com.mysql.jdbc.Driver"); - con = DriverManager.getConnection( - "jdbc:mysql://192.168.80.130:3306/ossean_production?useUnicode=true&characterEncoding=utf-8" - , "trustie", "1234");// 鍒涘缓鏁版嵁杩炴帴 - - } catch (Exception e) { - logger.info("连接错误" + e.getMessage()); - } - return con; - } - return con; - } - - public void getMatchRecord(){ - try { - Scanner in = new Scanner(record); - this.setPrjHistory(in.nextInt()); - this.setMemoHistory(in.nextInt()); - in.close(); - } catch (FileNotFoundException e) { - logger.info(e.getStackTrace().toString()); - } - } - - public void writeIntoRecord(int prjId, int memoId){ - try { - FileOutputStream fout = new FileOutputStream(record); - fout.write((prjId + "\t" + memoId).getBytes()); - fout.close(); - } catch (FileNotFoundException e) { - logger.info(e.getStackTrace().toString()); - } catch (IOException e) { - logger.info(e.getStackTrace().toString()); - } - } - - public void updateOspTags(Connection conn) { - logger.info("start updateOspTags ......"); - String ospSql = "SELECT id FROM open_source_projects WHERE id > ? AND id <= ?"; - String sql = "SELECT tags.`name` FROM taggings, tags WHERE taggings.taggable_id = ? AND taggings.taggable_type = 'OpenSourceProject' AND tags.id = taggings.tag_id"; - String deleteOspTagsSql = "UPDATE open_source_projects SET tags = '';"; - String updateOspSql = "UPDATE open_source_projects SET tags = ? WHERE id = ?;"; - int prjId = this.getPrjHistory(); - int end = prjId + 5000; - try { - PreparedStatement ps3 = conn.prepareStatement(deleteOspTagsSql); - ps3.executeUpdate(); - ps3.close(); - logger.info("已清空tags..."); - while (end < 20000) { - PreparedStatement ps2 = conn.prepareStatement(ospSql); - ps2.setInt(1, prjId); - ps2.setInt(2, end); - ResultSet pIds = ps2.executeQuery(); - if (!pIds.next()) { - prjId = end; - } else { - pIds.beforeFirst(); - } - while (pIds.next()) { - prjId = pIds.getInt(1); - PreparedStatement ps = conn.prepareStatement(sql); - ps.setInt(1, prjId); - ResultSet rs = ps.executeQuery(); - String temp_tag = ""; - while (rs.next()) { - temp_tag += "<" + rs.getString(1) + ">,"; - } - String curPrjTag = ""; - if (temp_tag.length() > 0) { - curPrjTag = temp_tag - .substring(0, temp_tag.length() - 1); - } - PreparedStatement ps1 = conn.prepareStatement(updateOspSql); - ps1.setString(1, curPrjTag); - ps1.setInt(2, prjId); - ps1.executeUpdate(); - conn.commit(); - rs.close(); - ps.close(); - ps1.close(); - logger.info("currentPrjId: " + prjId); - } - logger.info("last prj batch end, currentPrjId: " + prjId); - end = prjId + 5000; - } - - } catch (SQLException e) { - logger.info(e.getStackTrace().toString()); - } - } - - public void updateOspTagsForSearch(Connection conn) { - logger.info("start updateOspTagsForSearch ......"); - String ospSql = "SELECT id FROM open_source_projects WHERE id > ? AND id <= ?"; - String sql = "SELECT tags.`name` FROM taggings, tags WHERE taggings.taggable_id = ? AND taggings.taggable_type = 'OpenSourceProject' AND tags.id = taggings.tag_id AND taggings.disagree_num > 50"; - String deleteOspTagsSql = "UPDATE open_source_projects SET tags_for_search = '';"; - String updateOspSql = "UPDATE open_source_projects SET tags_for_search = ? WHERE id = ?;"; - int prjId = this.getPrjHistory(); - int end = prjId + 5000; - try { - PreparedStatement ps3 = conn.prepareStatement(deleteOspTagsSql); - ps3.executeUpdate(); - ps3.close(); - logger.info("已清空tags_for_search..."); - while (end < 400000) { - PreparedStatement ps2 = conn.prepareStatement(ospSql); - ps2.setInt(1, prjId); - ps2.setInt(2, end); - ResultSet pIds = ps2.executeQuery(); - if (!pIds.next()) { - prjId = end; - } else { - pIds.beforeFirst(); - } - while (pIds.next()) { - prjId = pIds.getInt(1); - PreparedStatement ps = conn.prepareStatement(sql); - ps.setInt(1, prjId); - ResultSet rs = ps.executeQuery(); - String temp_tag = ""; - while (rs.next()) { - temp_tag += "<" + rs.getString(1) + ">,"; - } - String curPrjTag = ""; - if (temp_tag.length() > 0) { - curPrjTag = temp_tag - .substring(0, temp_tag.length() - 1); - } - PreparedStatement ps1 = conn.prepareStatement(updateOspSql); - ps1.setString(1, curPrjTag); - ps1.setInt(2, prjId); - ps1.executeUpdate(); - conn.commit(); - rs.close(); - ps.close(); - ps1.close(); - logger.info("currentPrjId: " + prjId); - } - logger.info("last prj batch end, currentPrjId: " + prjId); - end = prjId + 5000; - } - - } catch (SQLException e) { - logger.info(e.getStackTrace().toString()); - } - } - - public void updateMemoTags(Connection conn) { - logger.info("start updateMemoTags ......"); - String memoSql = "SELECT id FROM relative_memos WHERE id > ? AND id <= ?"; - String sql = "SELECT tags.`name` FROM taggings, tags WHERE taggings.taggable_id = ? AND taggings.taggable_type = 'RelativeMemo' AND tags.id = taggings.tag_id"; - String updateMemoSql = "UPDATE relative_memos SET tags = ? WHERE id = ?"; - int memoId = this.getMemoHistory(); - int end = memoId + 5000; - try { - while (end < 4000000) { - PreparedStatement ps2 = conn.prepareStatement(memoSql); - ps2.setInt(1, memoId); - ps2.setInt(2, end); - ResultSet memoIds = ps2.executeQuery(); - if (!memoIds.next()) { - memoId = end; - } else { - memoIds.beforeFirst(); - } - - while (memoIds.next()) { - memoId = memoIds.getInt(1); - PreparedStatement ps = conn.prepareStatement(sql); - ps.setInt(1, memoId); - ResultSet rs = ps.executeQuery(); - String temp_tag = ""; - while (rs.next()) { - temp_tag += "<" + rs.getString(1) + ">,"; - } - String curMemoTag = ""; - if (temp_tag.length() > 0) { - curMemoTag = temp_tag.substring(0, - temp_tag.length() - 1); - } - PreparedStatement ps1 = conn - .prepareStatement(updateMemoSql); - ps1.setString(1, curMemoTag); - ps1.setInt(2, memoId); - ps1.executeUpdate(); - conn.commit(); - rs.close(); - ps.close(); - ps1.close(); - logger.info("currentMemoId: " + memoId); - } - logger.info("last memo batch end, currentMemoId: " + memoId); - end = memoId + 5000; - } - - } catch (SQLException e) { - logger.info(e.getStackTrace().toString()); - } - } - -} diff --git a/update_tags/src/main/java/OSSEAN/org/ossean/updatetags/Main.java b/update_tags/src/main/java/OSSEAN/org/ossean/updatetags/Main.java deleted file mode 100644 index cee1f2c77..000000000 --- a/update_tags/src/main/java/OSSEAN/org/ossean/updatetags/Main.java +++ /dev/null @@ -1,26 +0,0 @@ -package OSSEAN.org.ossean.updatetags; - -import java.sql.SQLException; -import com.mysql.jdbc.Connection; - -/** - * - * @author zhanyun - * - */ -public class Main { - - public static void main(String[] args) throws SQLException { - GetData getData = new GetData(); - Connection conn; - conn = (Connection) getData.getConnection(); - conn.setAutoCommit(false); - getData.updateOspTagsForSearch(conn); - getData.updateOspTags(conn); - conn.close(); - Connection conn1; - conn1 = (Connection) getData.getConnection(); - conn1.setAutoCommit(false); - getData.updateMemoTags(conn1); - } -} \ No newline at end of file diff --git a/update_tags/src/main/resource/log4j.xml b/update_tags/src/main/resource/log4j.xml deleted file mode 100644 index cad5b8ff1..000000000 --- a/update_tags/src/main/resource/log4j.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file