用HtmlUnit和httpClient抓施华洛世奇网站图片和动画<二>
try
????{
?????mainPage = webClient.getPage(url);
????} catch (Exception e)
????{
?????log.error(e.getMessage(), e);
????}
????if (mainPage != null)
????{
?????HtmlElement paginationContainer = mainPage
???????.getElementById("paginationContainer");
?????if (paginationContainer != null)
?????{
??????url = url + "/all-1";
??????try
??????{
???????mainPage = webClient.getPage(url);
??????} catch (Exception e)
??????{
???????log.error(e.getMessage(), e);
??????}
?????}
HtmlElement categories = mainPage.getElementById("categories");
?????if (categories == null)
?????{
??????HtmlElement products = mainPage.getElementById("products");
??????if(products!=null)
??????{
???????List<HtmlElement> productDivList = products.getElementsByTagName("div");
???????if(productDivList!=null&&productDivList.size()>0)
???????{
????????for(HtmlElement proDiv:productDivList)
????????{
?????????List<HtmlElement> subList = proDiv.getElementsByTagName("div");
?????????if(subList!=null)
?????????{
??????????for(HtmlElement dt:subList)
??????????{
???????????String classt = dt.getAttribute("class");
???????????if (classt != null && classt.equals("productImg"))
???????????{
????????????List<HtmlElement> subAList = proDiv.getElementsByTagName("a");
????????????if(subAList!=null&&subAList.size()>0)
????????????{
?????????????HtmlElement ae=subAList.get(0);
?????????????String href = ae.getAttribute("href");
?????????????thirdPageUrls.add(href);
????????????}
????????????break;
???????????}
??????????}
?????????}
????????}
???????}
??????}
?????}
?????else
?????{
????? List<HtmlElement> divList = categories.getElementsByTagName("div");
???????? if (divList != null && divList.size() > 0)
???????? {
???????for (HtmlElement div : divList)
???????{
????????List<HtmlElement> tempDivs = div
??????????.getElementsByTagName("div");
????????if (tempDivs != null && tempDivs.size() > 0)
????????{
?????????for (HtmlElement div1 : tempDivs)
?????????{
??????????List<HtmlElement> aList = div1
????????????.getElementsByTagName("a");
??????????HtmlAnchor a = (HtmlAnchor) aList
????????????.get(0);
??????????String link = a.getHrefAttribute();
??????????secondPageUrls.add(link);
?????????}
????????}
???????}
??????}
?????}
????
????}
???}
??}
??log.error("第二层抓取结束..........");
??log.error("目前抓取到的第二层URL个数为:"+secondPageUrls.size());
?
int count=0;
??
??for (String url : secondPageUrls)
??{
???count++;
???log.error("正在抓取第二层的第"+count+"个URL:"+url);
???HtmlPage mainPage = null;
???try
???{
????mainPage = webClient.getPage(url);
???} catch (Exception e)
???{
????log.error(e.getMessage(), e);
???}
???log.error("抓取URL完成:"+url+",正在分析URL"+url+"+结果的URL");
???if (mainPage != null)
???{
????HtmlElement paginationContainer = mainPage
??????.getElementById("paginationContainer");
????if (paginationContainer != null)
????{
?????url = url + "/all-1";
????}
????HtmlElement products = mainPage.getElementById("products");
????if (products != null)
????{
?????List<HtmlElement> list = products
???????.getElementsByTagName("div");
?????;
?????if (list == null || list.size() == 0)
?????{
??????continue;
?????}
?????for (HtmlElement h : list)
?????{
??????String cls = h.getAttribute("class");
??????if (cls == null || !cls.equals("productName"))
??????{
???????continue;
??????}
??????List<HtmlElement> links = h.getElementsByTagName("a");
??????if (links != null && links.size() > 0)
??????{
???????HtmlAnchor htmlAnchor = (HtmlAnchor) links.get(0);
???????String linkStr = htmlAnchor.getHrefAttribute();
???????thirdPageUrls.add(linkStr);
???????log.error(linkStr);
??????}
?????}
????}
???}
??}
??log.error("第二层抓取结束..........");
??secondPageUrls.clear();
??secondPageUrls = null;
??
?? count=0;
??log.error("目前抓取到的第三层URL个数为:"+thirdPageUrls.size());
??String urlPrix="http://www.swarovski-crystallized.com/jewelry/us/";
?for (String url : thirdPageUrls)
??{
???count++;
???log.error("正在抓取第三层的第"+count+"个URL:"+url);
???HtmlPage mainPage = null;
???try
???{
????mainPage = webClient.getPage(url);
???} catch (Exception e)
???{
????log.error(e.getMessage(), e);
???}
???if (mainPage != null)
???{
????log.error("抓取URL完成:"+url+",正在分析URL"+url+"+结果");
????int indexC=url.indexOf(urlPrix);
????int indexD=url.indexOf("?");
????String dirStr=url.substring(indexC+urlPrix.length(), indexD);
????String regEx = "/";
//????Pattern p = Pattern.compile(regEx);
//????Matcher m = p.matcher(dirStr);
????
//哦哦哦，建立文件夹准备把抓到数据放在里面
????dirStr=replece( regEx,"\\\",dirStr);
????dirStr = "D:\\swaroski\"+dirStr;
????File?? file?? =?? new?? File(dirStr);??
????if(file.isDirectory())
????{
?????dirStr=dirStr+"\"+count;
?????file?? =?? new?? File(dirStr);?
????}
?file.mkdirs();
Product product=new Product();
????product.setLocalDir(dirStr);
????
????product.setPageUrl(url);
????HtmlElement rightCol = mainPage.getElementById("rightCol");
????String title = null;
????String description = null;
????String packingUnit = null;
????if (rightCol != null)
????{
?????HtmlElement headlineDiv = rightCol
???????.getElementById("headline");
?????if (headlineDiv != null)
。。。。。。。。。。。。。。。。。。。。。。。。
log.error("完成:"+url+",分析结果");
?????try
?????{
??????swaroSkiDAO.addProduct(product);
?????} catch (Exception e)
?????{
??????log.error(e.getMessage(), e);
?????}
?????log.error("完成保存结果");
?????
?????for(String downloadUrl:resourceUrlList)
?????{
??????int index6=downloadUrl.lastIndexOf("/");
??????String fileName=downloadUrl.substring(index6+1);
??????String dirStr2=dirStr+"\"+fileName;
???????? File storeFile = new File(fileName);
???????? if(storeFile.exists())
???????? {
????????? continue;
???????? }
??????SaveFileThread runable=new SaveFileThread(dirStr2,downloadUrl,sem);
??????pools.submit(runable);
??????log.error("开始提交下载文件:"+downloadUrl);
??????try
??????{
???????Thread.sleep(2500);
??????} catch (InterruptedException e)
??????{
??????}
}
?
?
//啊哈哈，这个就是去拿下图片和动画的线程，结束
class SaveFileThread implements Runnable
?{
??private String fileName;
??private String downloadUrl;
??private Semaphore sem;
????????? public SaveFileThread(String fileName,String downloadUrl,Semaphore sem)
????????? {
??????? ?? this.fileName=fileName;
??????? ?? this.downloadUrl=downloadUrl;
??????? ?? this.sem=sem;
????????? }
??public void run()
??{
??????????? HttpClient client = new HttpClient();?
?????????? GetMethod get = new GetMethod(downloadUrl);?
?????????? FileOutputStream output=null;
?????????? try
?????{
??????? client.executeMethod(get);
???????? File storeFile = new File(fileName);?
???????????? output = new FileOutputStream(storeFile);?
???????????? output.write(get.getResponseBody());
???????????? output.flush();
?????} catch (Exception e)
?????{
??????log.error(e.getMessage(), e);
?????}
