Scraping Wikileaks for Hillary Clinton Emails related to Bangladesh

import requests, os, bs4, webbrowser, re, json
import pandas as pd

data = {
    'title' : [] ,
    'from' : [] ,
    'to': [] ,
    'date' : [] ,
    'subject': [],
    'body' : []

}

page = 1
base_url = "https://search.wikileaks.org/"
url = 'https://search.wikileaks.org/?query=bangladesh&exact_phrase=&any_of=&exclude_words=&document_date_start=&document_date_end=&released_date_start=&released_date_end=&publication_type%5B%5D=42&new_search=False&order_by=newest_released_date#results'
# There are 8 result pages, so we use this loop to go through each page and scrape.
while (page < 9):
    #set the url and go to it
    print "going to wikileaks for searching, page:", page
    res = requests.get(url)
    res.raise_for_status()


    # get the search result page
    soup = bs4.BeautifulSoup(res.text)
    linkElems = soup.select('.info a')

    #open each page and scrape data
    for i in range(len(linkElems)):

        # get a search result's content
        result_url = linkElems[i].get('href')
        result_html = requests.get(result_url)
        result_html.raise_for_status()
        result_soup = bs4.BeautifulSoup(result_html.text)

        #extract the data
        #title of the document
        title = result_soup.select('.tab-content h2')[0].get_text()
        #body of the mail
        content = result_soup.select('.email-content')[0].get_text()
        content = content.encode('utf-8')
        #Strip unnecessary white spaces
        content = re.sub(r'\s+ ', ' ', content)

        #get the from, to and subject data from header
        header = result_soup.select('#header')[0].get_text()
        # break down the header to from, to, date and subject fields to fit our dictionary format
        #get the sender
        sender = header.splitlines()[1]
        sender = sender.split(':')
        sender = sender[1].encode('utf-8')

        #get the receiver
        receiver = header.splitlines()[2]
        receiver = receiver.split(':')
        receiver = receiver[1].encode('utf-8')

        # get the date time
        dt = header.splitlines()[3]
        dt = dt.split(':')
        dt = dt[1].encode('utf-8') + dt[2].encode('utf-8')

        # get the subject
        subject = header.splitlines()[4]
        subject = subject.split(':')
        subject = subject[1].encode('utf-8')

        #add all the data to our dictionary
        data['title'].append(title)
        data['from'].append(sender)
        data['to'].append(receiver)
        data['date'].append(dt)
        data['subject'].append(subject)
        data['body'].append(content)


    #go to next page
    page += 1
    if page < 9:
        #get the next page's link
        next_page = soup.select('.next a')
        next_page_url = next_page[0].get('href')
        #set url
        url = base_url + next_page_url

#get the data into a pandas dataframe
email_leaks = pd.DataFrame(data)

email_leaks.shape

email_leaks

#Saving the data in different formats
#Latex
email_leaks.to_latex('leaks.tex')
#HTML
email_leaks.to_html('lix.html')
#JSON
email_leaks.to_json('mails.json')
#Text File
json.dump(data, open("mails.txt",'w'))