RomanGolienko
/
Roman_applicant-site
forked from aslan/applicant-site
1
1
Fork 0
Code Pull Requests 2 Activity

new Scrapper embodying

This commit is contained in:
ROMANGOLIENKO 2024-01-29 17:25:49 +03:00
parent 4c5c0e6ddb
commit b20dcc4ad8
18 changed files with 151 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
app/Services/PageScrapper.php
View File

@ -88,14 +88,47 @@ class PageScrapper
} }
public function normalizePath() public function normalizePath()
{ {
$links = (string) $this->document->find('a[href]'); $rez = $this->document;
dd($links); $content = $rez->first($this->contentMarker);
foreach ($links as $link) { $links = $content->find('a[href]');
if (!str_starts_with($link, 'https')) { $srclinks = $content->find('img[src]');
$this->document = str_replace($link, 'https://mkgtu.ru' . $link, $this->document);
}
}
return $this->document;
$html0 = $content->html();
foreach ($links as $k => $link) {
$href = $link->attr('href');
if (!str_contains($link->attr('href'), "https://")) {
$tmp = explode('/', rawurldecode($href));
foreach ( $tmp as $k => $v) {
$tmp[$k] = rawurlencode($v);
}
$href = implode('/', $tmp);
$html0 = str_replace($href, 'https://mkgtu.ru' . $href, $html0);
}
}
foreach ($srclinks as $k => $srclink) {
$src = $srclink->attr('src');
if (!str_contains($srclink->attr('src'), "https://")) {
$tmp = explode('/', rawurldecode($src));
foreach ( $tmp as $k => $v) {
$tmp[$k] = rawurlencode($v);
}
$src = implode('/', $tmp);
$html0 = str_replace($src, 'https://mkgtu.ru' . $src, $html0);
}
}
str_replace('st yle', 'style', $html0);
return $html0;
} }
} }

22
resources/views/layouts/applicant-layout.blade.php
View File

@ -319,9 +319,27 @@
</div> </div>
</div> </div>
</div> </div>
</div>
<!-- brnad section end -->
</div>
<!-- brnad section end -->
<style>
body {
font-weight: 300;
font-size: 16px;
line-height: 140%;
color: #222222;
margin: 12px 0;
text-align: justify;
}
a {
color: #005951;
}
</style>
@yield('content') @yield('content')
<!-- footer area start --> <!-- footer area start -->

12
resources/views/menu/abitur/aspirantura.blade.php
View File

@ -1,15 +1,15 @@
@extends('layouts.applicant-layout') @extends('layouts.applicant-layout')
@section('content') @section('content')
<div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Для поступающих на обучение по программам аспирантуры </div> <div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Онлайн-консультации для поступающих и родителей </div>
<div class="container pt-4 d-flex justify-content-center"> <div class="container fs-5 pt-4 d-flex justify-content-center">
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/aspirantura/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://www.mkgtu.ru/postuplenie/web-consultations/",".content_info");
$row = $pageScrapper->getHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp

7
resources/views/menu/abitur/kolledzh.blade.php
View File

@ -6,10 +6,9 @@
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/kolledzh/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/kolledzh/",".content_info");
$row = $pageScrapper->getHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp

7
resources/views/menu/abitur/ordinatura.blade.php
View File

@ -6,12 +6,9 @@
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/ordinatura/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/ordinatura/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->normalizeURLFile($row);
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp
</div> </div>

6
resources/views/menu/abitur/platnye-obrazovatelnye-uslugi.blade.php
View File

@ -6,10 +6,8 @@
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/platnye-obrazovatelnye-uslugi/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/platnye-obrazovatelnye-uslugi/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->normalizeURLFile($row);
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp

6
resources/views/menu/abitur/spetsialitet-magistratura.blade.php
View File

@ -6,10 +6,8 @@
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/spetsialitet-magistratura-2022/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/spetsialitet-magistratura-2022/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->normalizeURLFile($row);
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp

11
resources/views/menu/abitur/videomaterialy-dlya-postupayushchikh.blade.php
View File

@ -6,15 +6,8 @@
<div class="col-10 text-center pb-4 "> <div class="col-10 text-center pb-4 ">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/videomaterialy-dlya-postupayushchikh/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/priemnaya-kampaniya/videomaterialy-dlya-postupayushchikh/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->normalizeURLFile($row);
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
// $content = str_replace(
// '<img width="614" alt="Обложка Госуслуги_page-0001.jpg" src="/upload/medialibrary/c76/c761087fd6938bd8eb8708e9e036679e.jpg" height="346" title="Обложка Госуслуги_page-0001.jpg">',
// '<img width="614" alt="Обложка Госуслуги_page-0001.jpg" src="https://mkgtu.ru/upload/medialibrary/c76/c761087fd6938bd8eb8708e9e036679e.jpg" height="346" title="Обложка Госуслуги_page-0001.jpg" class="border border-3 rounded-3 border-secondary">',
// $content);
echo $content; echo $content;
@endphp @endphp

6
resources/views/menu/abitur/web-consultations.blade.php
View File

@ -6,10 +6,8 @@
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://www.mkgtu.ru/postuplenie/web-consultations/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://www.mkgtu.ru/postuplenie/web-consultations/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->normalizeURLFile($row);
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp

22
resources/views/menu/course.blade.php
View File

@ -36,28 +36,12 @@
<div class="col-10"> <div class="col-10">
@php @php
use DiDom\Document;
$document = new Document('https://mkgtu.ru/postuplenie/podat-dokumenty-onlayn/', true);
$posts = $document->find('.content_info');
echo $document->first('.content_info')->html();
/*
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/podat-dokumenty-onlayn/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/podat-dokumenty-onlayn/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->printHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
*/
@endphp @endphp
</div> </div>
</div> </div>

36
resources/views/menu/inostrannym-abiturientam/akademicheskaya-mobilnost-i-mezhdunarodnoe-sotrudnichestvo.blade.php
View File

@ -6,34 +6,18 @@
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/akademicheskaya-mobilnost-i-mezhdunarodnoe-sotrudnichestvo/akademicheskaya-mobilnost-i-fondy/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/akademicheskaya-mobilnost-i-mezhdunarodnoe-sotrudnichestvo/akademicheskaya-mobilnost-i-fondy/",".content_info");
$PageScrapper2 = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/akademicheskaya-mobilnost-i-mezhdunarodnoe-sotrudnichestvo/individualnaya-mobilnost-studentov-i-aspirantov/",".content_info");
$PageScrapper3 = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/akademicheskaya-mobilnost-i-mezhdunarodnoe-sotrudnichestvo/kak-stat-uchastnikom-programm-akademicheskoy-mobilnosti/",".content_info");
$row = $pageScrapper->getHTML(); $content = '<div class=" fw-bolder fs-3 text-center py-5 lh-lg"> Академическая мобильность и фонды </div>';
//$tmp = explode('<footer',$row); $row = $tmp[0]; $content .= $PageScrapper->normalizePath();
$row = $pageScrapper->cutHTML($row,'/<footer(.*)<\/footer>/isU'); $content .= '<div class=" fw-bolder fs-3 text-center py-5 lh-lg"> Индивидуальная мобильность студентов и аспирантов </div>';
$content .= $PageScrapper2->normalizePath();
$content .= '<div class=" fw-bolder fs-3 text-center py-5 lh-lg"> Как стать участником программ академической мобильности </div>';
$pageScrapper2 = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/akademicheskaya-mobilnost-i-mezhdunarodnoe-sotrudnichestvo/individualnaya-mobilnost-studentov-i-aspirantov/", '<div class=["\']content_info["\']>'); $content .= $PageScrapper3->normalizePath();
$row2 = $pageScrapper2->getHTML();
//$tmp = explode('<footer',$row2); $row2 = $tmp[0];
$row2 = $pageScrapper2->cutHTML($row2,'/<footer(.*)<\/footer>/isU');
$pageScrapper3 = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/akademicheskaya-mobilnost-i-mezhdunarodnoe-sotrudnichestvo/kak-stat-uchastnikom-programm-akademicheskoy-mobilnosti/", '<div class=["\']content_info["\']>');
$row3 = $pageScrapper3->getHTML();
//$tmp = explode('<footer',$row3); $row3 = $tmp[0];
$row3 = $pageScrapper3->cutHTML($row3,'/<footer(.*)<\/footer>/isU');
$content = $row . $row2 . $row3;
//dd($content);
//$content = $pageScrapper->cutHTML($row,'/<footer(.*)<\/footer>/isU');
// $content = $pageScrapper->normalizeURLFile($content);
echo $content; echo $content;

8
resources/views/menu/inostrannym-abiturientam/kafedry.blade.php
View File

@ -1,15 +1,15 @@
@extends('layouts.applicant-layout') @extends('layouts.applicant-layout')
@section('content') @section('content')
<div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Кафедры </div> <div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Кафедры </div>
<div class="container py-4 d-flex justify-content-center"> <div class="container py-4 d-flex justify-content-center">
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/kafedry/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/kafedry/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->printHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp

17
resources/views/menu/inostrannym-abiturientam/mezhdunarodnaya-deyatelnost.blade.php
View File

@ -1,15 +1,21 @@
@extends('layouts.applicant-layout') @extends('layouts.applicant-layout')
@section('content') @section('content')
<style>
a {
color: #005951;
}
</style>
<div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Международная деятельность </div> <div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Международная деятельность </div>
<div class="container py-4 d-flex justify-content-center"> <div class="container fs-5 pt-4 d-flex justify-content-center">
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/mezhdunarodnaya-deyatelnost/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/mezhdunarodnaya-deyatelnost/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->printHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp
@ -17,3 +23,4 @@
</div> </div>
</div> </div>
@endsection @endsection

16
resources/views/menu/inostrannym-abiturientam/obshchie-svedeniya.blade.php
View File

@ -1,15 +1,19 @@
@extends('layouts.applicant-layout') @extends('layouts.applicant-layout')
@section('content') @section('content')
<style>
a {
color: #005951;
}
</style>
<div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Общие сведения </div> <div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Общие сведения </div>
<div class="container py-4 d-flex justify-content-center"> <div class="container fs-5 pt-4 d-flex justify-content-center">
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/obshchie-svedeniya/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/obshchie-svedeniya/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->printHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp
@ -17,3 +21,5 @@
</div> </div>
</div> </div>
@endsection @endsection

8
resources/views/menu/inostrannym-abiturientam/tsentr-mezhdunarodnogo-obrazovaniya.blade.php
View File

@ -1,15 +1,15 @@
@extends('layouts.applicant-layout') @extends('layouts.applicant-layout')
@section('content') @section('content')
<div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Центр Международного образования </div> <div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Центр Международного образования </div>
<div class="container py-4 d-flex justify-content-center"> <div class="container py-4 d-flex justify-content-center">
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/tsentr-mezhdunarodnogo-obrazovaniya/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/inostrannym-abiturientam/tsentr-mezhdunarodnogo-obrazovaniya/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->printHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp

20
resources/views/menu/olimpiady-dlya-shkolnikov.blade.php
View File

@ -1,5 +1,8 @@
@extends('layouts.applicant-layout') @extends('layouts.applicant-layout')
@section('content') @section('content')
<div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Олимпиады для школьников </div> <div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Олимпиады для школьников </div>
{{-- пофиксить!!!!--}} {{-- пофиксить!!!!--}}
<div class="container py-4 d-flex justify-content-center" style="padding-left: 150px;" > <div class="container py-4 d-flex justify-content-center" style="padding-left: 150px;" >
@ -7,20 +10,9 @@
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/olimpiady-dlya-shkolnikov/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/olimpiady-dlya-shkolnikov/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->printHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
// $tmp = preg_match_all('/<p (.*)>/isU', "https://mkgtu.ru/postuplenie/olimpiady-dlya-shkolnikov/", $arr);
//
// $arr[0] = array_unique($arr[0]);
// $str = $arr[0][0];
$content = str_replace('<p style="text-align: left;">', '<p>', $content);
echo $content; echo $content;
@endphp @endphp

9
resources/views/menu/paid_edu.blade.php
View File

@ -5,7 +5,7 @@
<link rel="stylesheet" type="text/css" href="{{ URL::to('https://mkgtu.ru/sveden/assets/review/v1/common/css/vendor.css?1694691653153304') }}"> <link rel="stylesheet" type="text/css" href="{{ URL::to('https://mkgtu.ru/sveden/assets/review/v1/common/css/vendor.css?1694691653153304') }}">
<link rel="stylesheet" type="text/css" href="{{ URL::to('https://mkgtu.ru/vikon/sveden/assets/review/v1/common/css/vendor.css?1706157854153304') }}"> <link rel="stylesheet" type="text/css" href="{{ URL::to('https://mkgtu.ru/vikon/sveden/assets/review/v1/common/css/vendor.css?1706157854153304') }}">
<link rel="stylesheet" type="text/css" href="{{ URL::to('https://mkgtu.ru/vikon/sveden/assets/review/v1/common/css/sveden.css?170615785463681') }}"> <link rel="stylesheet" type="text/css" href="{{ URL::to('https://mkgtu.ru/vikon/sveden/assets/review/v1/common/css/sveden.css?170615785463681') }}">
<link rel="stylesheet" type="text/css" href="{{ URL::to('https://mkgtu.ru/dist/css/style.css?1705561951294124') }}">
@endsection @endsection
@section('content') @section('content')
@ -15,10 +15,9 @@
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/sveden/paid_edu/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/sveden/paid_edu/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->printHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp

9
resources/views/menu/podgotovitelnye-kursy.blade.php
View File

@ -2,15 +2,14 @@
@section('content') @section('content')
<div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Подготовительные курсы </div> <div class=" fw-bolder fs-1 text-center py-5 lh-lg"> Подготовительные курсы </div>
<div class="container pt-4 d-flex justify-content-center"> <div class="container fs-5 pt-4 d-flex justify-content-center">
<div class="col-10"> <div class="col-10">
@php @php
use App\Services\PageScrapper; use App\Services\PageScrapper;
$pageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/podgotovitelnye-kursy/", '<div class=["\']content_info["\']>'); $PageScrapper = new PageScrapper("https://mkgtu.ru/postuplenie/podgotovitelnye-kursy/",".content_info");
$row = $pageScrapper->getHTML(); $content = $PageScrapper->printHTML();
$content = $pageScrapper->normalizeURLFile($row); $content = $PageScrapper->normalizePath();
$content = $pageScrapper->cutHTML($content,'/<footer(.*)<\/footer>/isU');
echo $content; echo $content;
@endphp @endphp