bg-mountains-scraping-routes/scrape.ipynb

In [7]:

def get_links_from_page(url):
    import requests
    from bs4 import BeautifulSoup
    import re

    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')

    # 2. Find all <a> or <button> tags containing the target text
    # We use re.IGNORECASE to catch variations like "ИЗТЕГЛЯНЕ"
    target_text = "Изтегляне"
    elements = soup.find_all(['a', 'button'], string=re.compile(target_text, re.IGNORECASE))

    download_links = []

    for tag in elements:
        # If the tag is directly an <a>, get its href
        if tag.name == 'a' and tag.get('href'):
            download_links.append(tag['href'])
        
        # If it's a <button>, it might be wrapped in an <a> tag
        elif tag.name == 'button':
            parent_a = tag.find_parent('a')
            if parent_a and parent_a.get('href'):
                download_links.append(parent_a['href'])

    # # Print results
    # for link in set(download_links): # 'set' removes duplicates
    #     print(f"Found download link: {link}")
    return download_links

In [12]:

import os
import sys
import time
import random

start_num = 0 
iter_num = 5
string_plchldr = "number_start_placeholder"

link = f"https://www.bgmountains.org/bg/gps-tracks/cats/all?type=all&start=number_start_placeholder"
lim = 172
#take from website
list_links = []
output_links = []
with open("out.log","w") as f:
    for i in range(lim):
        print("Page: ",i+1)
        response_links = get_links_from_page(link.replace(string_plchldr,str(iter_num*i)))
        output_links.append(response_links)
        time.sleep(random.randint(0,6))
        f.write(str(response_links) + "\n")

Page:  1
Page:  2
Page:  3
Page:  4
Page:  5
Page:  6
Page:  7
Page:  8
Page:  9
Page:  10
Page:  11
Page:  12
Page:  13
Page:  14
Page:  15
Page:  16
Page:  17
Page:  18
Page:  19
Page:  20
Page:  21
Page:  22
Page:  23
Page:  24
Page:  25
Page:  26
Page:  27
Page:  28
Page:  29
Page:  30
Page:  31
Page:  32
Page:  33
Page:  34
Page:  35
Page:  36
Page:  37
Page:  38
Page:  39
Page:  40
Page:  41
Page:  42
Page:  43
Page:  44
Page:  45
Page:  46
Page:  47
Page:  48
Page:  49
Page:  50
Page:  51
Page:  52
Page:  53
Page:  54
Page:  55
Page:  56
Page:  57
Page:  58
Page:  59
Page:  60
Page:  61
Page:  62
Page:  63
Page:  64
Page:  65
Page:  66
Page:  67
Page:  68
Page:  69
Page:  70
Page:  71
Page:  72
Page:  73
Page:  74
Page:  75
Page:  76
Page:  77
Page:  78
Page:  79
Page:  80
Page:  81
Page:  82
Page:  83
Page:  84
Page:  85
Page:  86
Page:  87
Page:  88
Page:  89
Page:  90
Page:  91
Page:  92
Page:  93
Page:  94
Page:  95
Page:  96
Page:  97
Page:  98
Page:  99
Page:  100
Page:  101
Page:  102
Page:  103
Page:  104
Page:  105
Page:  106
Page:  107
Page:  108
Page:  109
Page:  110
Page:  111
Page:  112
Page:  113
Page:  114
Page:  115
Page:  116
Page:  117
Page:  118
Page:  119
Page:  120
Page:  121
Page:  122
Page:  123
Page:  124
Page:  125
Page:  126
Page:  127
Page:  128
Page:  129
Page:  130
Page:  131
Page:  132
Page:  133
Page:  134
Page:  135
Page:  136
Page:  137
Page:  138
Page:  139
Page:  140
Page:  141
Page:  142
Page:  143
Page:  144
Page:  145
Page:  146
Page:  147
Page:  148
Page:  149
Page:  150
Page:  151
Page:  152
Page:  153
Page:  154
Page:  155
Page:  156
Page:  157
Page:  158
Page:  159
Page:  160
Page:  161
Page:  162
Page:  163
Page:  164
Page:  165
Page:  166
Page:  167
Page:  168
Page:  169
Page:  170
Page:  171
Page:  172

In [9]:

output_links

Out [9]:

[['/bg/gps-tracks/cats/summary/4-belasitsa/1-biloto',
  '/bg/gps-tracks/cats/summary/3-osogovo-belasishka/5-dzhama-kadiitsa',
  '/bg/gps-tracks/cats/summary/33-pirin/144-sandanski-lilyanovo',
  '/bg/gps-tracks/cats/summary/6-osogovo/7-trite-buki-ruen',
  '/bg/gps-tracks/cats/summary/8-maleshevska/11-krupnik']]

In [17]:

flat = []

with open("url-final.txt","w") as f:
    for item in output_links:
        for entry in item:
            flat.append(entry)
            f.write("https://www.bgmountains.org"+entry+"\n")

In [ ]:

858

858