java处置海量数据的初步解决思路

java处理海量数据的初步解决思路

在开发过程中经常遇到，海量数据处理的问题，最近，在工作中遇到给100W用户同事发送邮件，要求用JAVA去实现，由于数据量比较大，内存和服务器CUP都抗不住，为此必须解决这一些列的问题，于是我初步想出一个解决该问题的方法~

???? 废话少说，晒上代码：

???


package oracle;import java.sql.Connection;import java.sql.ResultSet;import java.sql.Statement;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;public class jdbcHelper {private static Connection con1, con2;private static Statement sta;private static ResultSet rs;/**  * 分页结果集  * @param sql  * @param start  * @param end  * @param id  * @return  */public static List queryList(String sql, int start, int end, Map params) {  List result=new ArrayList();  try {   con1 = JdbcUtils.getConnection();// 获得数据库连接   StringBuffer buffer = new StringBuffer();   buffer.append(&quot;select * from (select row_.*, rownum rownum_ from ( &quot;);   buffer.append(sql);   buffer.append(&quot; ) row_ where rownum &quot;);   buffer.append(start);   if(params.size()&gt;0){    buffer.append(&quot; and id=&quot;).append("'").append(params.get("id")).append("'");   }   sta = con1.createStatement();   rs = sta.executeQuery(buffer.toString());   while (rs.next()) {    result.add(rs.getString(1));   }  } catch (Exception e) {   e.printStackTrace();  } finally {   JdbcUtils.closeConnection(rs, sta, con1);  }  return result;}/**  * 总数  * @param sql  * @param id  * @return  */public static int getCount(String sql,Map params){   int count=0;  try {   con1 = JdbcUtils.getConnection();// 获得数据库连接   StringBuffer buffer = new StringBuffer();   buffer.append(sql);   if(params.size()&gt;0){    buffer.append(&quot; where id=&quot;).append(&quot;'&quot;).append(params.get(&quot;id&quot;)).append(&quot;'&quot;);   }   sta = con1.createStatement();   rs = sta.executeQuery(buffer.toString());   while (rs.next()) {    count=rs.getInt(1);   }  } catch (Exception e) {   e.printStackTrace();  } finally {   JdbcUtils.closeConnection(rs, sta, con1);  }  return count; }}上面这段代码主要实现ORCAL数据库分页，目的在于用多少取多少！package oracle;import java.util.List;import java.util.Map;/** * . * * @author dongwenhua * */public class PageList { private int curPage; // 当前是第几页 private int maxPage; // 一共有多少页 private int maxRowCount; // 一共有多少行 private int rowsPerPage = 10;// 每页多少行 private Object data; private int startPage; private int endPage;  public int getStartPage() {  return startPage; } public void setStartPage(int startPage) {  this.startPage = startPage; } public int getEndPage() {  return endPage; } public void setEndPage(int endPage) {  this.endPage = endPage; } public void setCurPage(int c) {  this.curPage = c ; } public void setMaxPage() { // 根据总行数计算总页数  if (this.maxRowCount % this.rowsPerPage == 0) {   this.maxPage = this.maxRowCount / this.rowsPerPage;  } else {   this.maxPage = this.maxRowCount / this.rowsPerPage + 1;  } } public void setMaxRowCount(int m) {  this.maxRowCount = m; } public int getCurPage() {  return this.curPage ; } public int getMaxPage() {  return this.maxPage; } public int getMaxRowCount() {  return this.maxRowCount; }  public int getRowsPerPage() {  return this.rowsPerPage; } public void setMaxPage(int maxPage) {  this.maxPage = maxPage; } public void setRowsPerPage(int rowsPerPage) {  this.rowsPerPage = rowsPerPage; } public Object getData() {  return data; } public void setData(Object data) {  this.data = data; } }这个内处理分页和结果集的封装package oracle;import java.util.HashMap;import java.util.List;import java.util.Map;/** * 解决大数据量发邮件问题 * * @author dongwenhua * */public class DataTest {    private static int no=0;  public static void main(String[] args) {  String sql = "select * from test ";  String count = "select count(1) from test";  Map params=new HashMap();//存放参数  int pageSize=10;  int total= jdbcHelper.getCount(count, params);//总数  int pageNo=1;  while(true){     if (no == total) { no=0;   break;   }   PageList list = getResultList(sql, count, params, pageNo, pageSize);   sendMail(list.getData());   try {    Thread.sleep(1000);   } catch (InterruptedException e) {    e.printStackTrace();   }   pageNo++;     }      System.out.println("other programm is run contiue"); }  public static PageList getResultList(String querySql,String countSql,Map params, int pageNo,int pageSize){  PageList page=new PageList();  page.setCurPage(pageNo);//当前页  page.setRowsPerPage(pageSize);//每页显示多少条  // 计数  int total = jdbcHelper.getCount(countSql, params);  page.setMaxRowCount(total);  page.setMaxPage();//计算出共有多少页  if(pageNo>page.getMaxPage()){   pageNo = page.getMaxPage();  }  int start = (pageNo - 1) * pageSize;  int end = pageNo * pageSize;  // 取单页结果集  List data = jdbcHelper.queryList(querySql, start, end, params);  page.setData(data);  return page;   }public static void sendMail(Object obj){ if(obj instanceof List){  List list=(List)obj;  for(int i=0;i<list.size();i++){   System.out.println("发送邮件--:"+list.get(i));   no++;  }    System.out.println("--end--"); }}}

?递归实现发送邮件