JAVA에서 NAVER 웹툰 크롤링후 DB에 저장하기

package crawling;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;

public class connection {
    Connection conn;

	public connection() {

		try {
			// mariaDB 드라이버 설정
			Class.forName("org.mariadb.jdbc.Driver");

			// 접속할 DB주소 설정
			String url = "jdbc:mysql://데이터베이스URL/DB명";

			// 접속할 아이디, 비밀번호 설정
			conn = DriverManager.getConnection(url, "아이디", "비밀번호");
			System.out.println("DB 연결 성공");

		} catch (ClassNotFoundException e){
			System.out.println("mariaDB Driver를 찾을수 없습니다.");
		} catch (SQLException e) {
			System.out.println("Database 연결중 에러가 발생 했습니다.");
		}
	}

	public PreparedStatement prepareStatement(String sql) throws SQLException {
		return conn.prepareStatement(sql);
	}

	public boolean isClosed() throws SQLException {
		return conn.isClosed();
	}

	public void close() throws SQLException {
		conn.close();
	}
}

package crawling;

import java.sql.PreparedStatement;
import java.sql.SQLException;

public class insert {

	public void insertQuery(int day, String title, String url, String thumb) {
		connection conn = new connection();
		PreparedStatement pstmt = null;

		try {
			// Insert문 ?부분은 아래의 입력값이 자동으로 변환이 됩니다.
			String sql = " INSERT INTO webtoon (no, day, title, url, thumb) VALUES (0, ?, ?, ?, ?) ";
			pstmt = conn.prepareStatement(sql);

			int index = 1;
			pstmt.setInt(index++, day);
			pstmt.setString(index++, title);
			pstmt.setString(index++, url);
			pstmt.setString(index++, thumb);

			// SQL실행
			int count = pstmt.executeUpdate();

			if (count == 0) {
				System.out.println("데이터값에 이상이 있습니다.");
			} else {
				System.out.println("데이터 Insert 성공!");
			}
		} catch (SQLException e) {
			System.out.println("Database 연결중 에러가 발생 했습니다.");
		} finally {
			try {
				if (conn != null && !conn.isClosed()) {
					conn.close();
				}
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}
	}
}

package crawling;

import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;

public class select {

	public int selectQuery(String title) {
    int result = 0;

		connection conn = new connection();
		PreparedStatement pstmt = null;
    ResultSet rs = null;


		try {
			// Select 쿼리문
			String sql = " SELECT count(*) as cnt FROM webtoon where title = ? ";
			pstmt = conn.prepareStatement(sql);

			int pstmtIndex = 1;
			pstmt.setString(pstmtIndex++, title);

			// SQL실행
			rs = pstmt.executeQuery();

			// 결과값 출력
			while (rs.next()) {
				result = rs.getInt(1);
				break;
			}
		} catch (SQLException e) {
			System.out.println("Database 연결중 에러가 발생 했습니다.");
		} finally {
			try {
				if (conn != null && !conn.isClosed()) {
					conn.close();
				}
			} catch (SQLException e) {
				e.printStackTrace();
			}
		}

		return result;
	}
}

package crawling;

import java.io.IOException;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

public class webtoon {

	public static void main(String[] args) {
		select selectSql = new select();
		insert insertSql = new insert();

		Document doc = null;
		Elements elemList = new Elements();
		String URL = "https://comic.naver.com/webtoon/weekday.nhn";
		String[] dateArr = {"h4.mon","h4.tue","h4.wed","h4.thu","h4.fri","h4.sat","h4.sun"};

		try {
			Connection connection = Jsoup.connect(URL)
					.userAgent("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36")
					.header("Accept", "text/html")
	        .header("Accept-Encoding", "gzip,deflate")
	        .header("Accept-Language", "it-IT,en;q=0.8,en-US;q=0.6,de;q=0.4,it;q=0.2,es;q=0.2")
	        .header("Connection", "keep-alive")
	        .ignoreContentType(true)
	        .timeout(30000);
			doc = connection.get();
			elemList = doc.select(".col_inner");

			if (elemList.size() > 0) {

				for (int i = 0; i < elemList.size(); i++) {
					Elements elems = elemList.get(i).children();

					if (elems.size() > 0) {

						for (int j = 0; j < dateArr.length; j++) {
							Elements elemTitle = new Elements();
							Elements elemUl = new Elements();
							elemTitle = elems.select(dateArr[j]);

							if (elemTitle != null && !elemTitle.isEmpty()) {
								System.out.println("------------------");
								System.out.println(elemTitle.text());
								System.out.println("------------------");

								elemUl = elems.select("ul > li");
								if (elemUl.size() > 0) {

									for (int k = 0; k < elemUl.size(); k++) {
										Elements elemLi = elemUl.get(k).children();

										if (selectSql.selectQuery(elemLi.text()) < 1) {
											Elements elemHref = elemLi.select(".thumb > a");
											Elements elemImg = elemLi.select(".thumb > a > img");

											System.out.println(elemLi.text());
											System.out.println(elemImg.attr("src"));
											System.out.println("------------------");

											insertSql.insertQuery(j, elemLi.text(), elemHref.attr("href"), elemImg.attr("src"));
										}
									}
								}
							}
						}
					}
				}
			}

		} catch (IOException e) {
			e.printStackTrace();
		}
	}

}