IadbModel = $IadbModel; $this->LoginUserLib = $LoginUserLib; $this->client = new Client(); $guzzleClient = new \GuzzleHttp\Client(array( 'curl' => array( CURLOPT_TIMEOUT => 500, CURLOPT_CONNECTTIMEOUT => 500, CURLOPT_FOLLOWLOCATION => false ), 'allow_redirects' => false, )); $this->client->setClient($guzzleClient); $this->client->setMaxRedirects(1); } /*----------- | Build Tender Data ---------------------*/ public function build_tender_data() { $process_records = 0; $per_batch = 20; $total_records = 0; $directory_arr = []; try { /*Pagination Call Request*/ $crawler = $this->client->request('GET','https://www.iadb.org/en/procurement-notices-search?query%5Bquery%5D=&query%5BprojectCountry%5D=&query%5Bproject_number%5D=&query%5BdateFrom_plan%5D=&query%5BdateTo_plan%5D=&query%5BNoticesType%5D=SPECIFIC&query%5BNotices%5D=CURRENT&page=0'); if(isset($crawler) && $crawler->count()) { if($crawler->filter('div#block-iadb-content > div.group-left > div.results-container > div.results')->count()) { $pagination_text = trim($crawler->filter('div#block-iadb-content > div.group-left > div.results-container > div.results')->text()); if(!empty($pagination_text)) { $arr_pagination_text = explode('Of',$pagination_text); if(!empty($arr_pagination_text)){ $total_records = end($arr_pagination_text); $total_records = str_replace(' Results', '',trim($total_records)); $total_records = intval($total_records); } } } } /*Call Request to Data*/ if($total_records>0) { $offset = ceil($total_records/$per_batch); for ($current_batch=0; $current_batch <=$offset; $current_batch++) { try { $crawler = $this->client->request('GET','https://www.iadb.org/en/procurement-notices-search?query%5Bquery%5D=&query%5BprojectCountry%5D=&query%5Bproject_number%5D=&query%5BdateFrom_plan%5D=&query%5BdateTo_plan%5D=&query%5BNoticesType%5D=SPECIFIC&query%5BNotices%5D=CURRENT&page='.$current_batch); if(isset($crawler) && $crawler->count()) { if($crawler->filter('div#block-iadb-content > div.group-left > div.results-container > div.results-table > table.table-links > tbody > tr')->count()) { $crawler->filter('div#block-iadb-content > div.group-left > div.results-container > div.results-table > table.table-links > tbody > tr')->each(function($node,$i) use(&$directory_arr,&$process_records){ if($node->count() && $node->filter('td')->count()) { $directory_arr = []; $node->filter('td')->each(function($list,$k) use(&$directory_arr){ if($list->count()) { /*Country*/ if($k==1){ $directory_arr['country'] = trim($list->text()); } /*Tender Title*/ if($k==2 && $list->filter('a')->count()){ $directory_arr['original_tender_title'] = trim($list->filter('a')->text()); $directory_arr['tender_url'] = trim($list->filter('a')->attr('href')); } /*Project Name*/ if($k==3 && $list->filter('a')->count()){ $directory_arr['project_name'] = trim($list->filter('a')->text()); } /*Invitation Date*/ if($k==4){ $invitation_date = trim($list->text()); $invitation_date = str_replace(',','',trim($invitation_date)); $directory_arr['invitation_date'] = !empty($invitation_date)?date('Y-m-d',strtotime($invitation_date)):''; } /*Closing Date*/ if($k==5){ $closing_date = trim($list->text()); $closing_date = str_replace(',','',trim($closing_date)); $directory_arr['closing_date'] = !empty($closing_date)?date('Y-m-d',strtotime($closing_date)):''; } } }); /*Insert Data*/ if(sizeof($directory_arr)>0) { if(isset($directory_arr['country']) && !empty($directory_arr['country'])){ if($directory_arr['country']=='Regional'){ $directory_arr['country'] = 'Worldwide'; $directory_arr['is_updated'] =1; $directory_arr['tender_title'] = !empty($directory_arr['original_tender_title'])?$directory_arr['original_tender_title']:''; } if($directory_arr['country']=='Trinidad and Tobago') { $directory_arr['is_updated'] =1; $directory_arr['tender_title'] = !empty($directory_arr['original_tender_title'])?$directory_arr['original_tender_title']:''; } } if(!empty($directory_arr['closing_date']) && $directory_arr['closing_date']>date('Y-m-d') && !empty($directory_arr['original_tender_title'])) { $check_tender_duplication = $this->is_tender_exist($directory_arr['original_tender_title']); if($check_tender_duplication) { $resp = $this->IadbModel->insert($directory_arr); if($resp){ $process_records++; } } } } } }); } } } catch(\Exception $e){ echo $e->getMessage(); } } } echo"[total_process_records=>".$process_records."]";exit; } catch(\Exception $e){ echo $e->getMessage();exit; } } /*---------- | Build Tender Detail Data ------------------*/ public function build_tender_detail_data() { ini_set('max_execution_time', 0); $process_records = 0; $arr_tenders = $this->IadbModel->where('process_status',0) ->whereRaw('DATE(created_at)="'.date('Y-m-d').'"') ->whereNotNull('original_tender_title') ->where('is_updated',0) ->get() ->toArray(); if(sizeof($arr_tenders)>0) { foreach ($arr_tenders as $key => $value) { if(!empty($value['original_tender_title'])) { try { /*Translate Language*/ $translated_arr = $this->build_language_translation($value['original_tender_title']); if(!empty($translated_arr)) { if(isset($translated_arr['tender_title']) && !empty($translated_arr['tender_title']) && $value['original_tender_title']!=$translated_arr['tender_title']) { $resp = $this->IadbModel->where('id',$value['id'])->update(['tender_title'=>$translated_arr['tender_title'],'is_updated'=>1]); if($resp){ $process_records++; } } } } catch(\Exception $e){ echo $e->getMessage(); } } } echo"[total_process_records=>".$process_records."]";exit; } else{ echo"No Records Found";exit; } } /*----------- | Build Tender Document ----------------*/ public function build_tender_document() { $process_records = 0; $arr_tenders = $this->IadbModel->where('process_status',0) ->whereRaw('DATE(created_at)="'.date('Y-m-d').'"') ->whereNotNull('tender_url') ->where('is_updated',1) ->where('is_document',1) ->whereIn('id',[1,2,3]) ->get() ->toArray(); if(sizeof($arr_tenders)>0) { foreach ($arr_tenders as $key => $value) { if(!empty($value['tender_url'])) { try { $name = $file_mimie_type =''; $ch = curl_init(); $source =str_replace(' ','%20',$value['tender_url']); curl_setopt($ch, CURLOPT_URL, $source); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false); curl_setopt($ch, CURLOPT_HEADER, false); $data = curl_exec($ch); $err = curl_error($ch); curl_close ($ch); if ($err) { echo "cURL Error #:" . $err; } elseif(sizeof($data)>0) { /*------------ | Build File ---------------------*/ $destination = '/opt/lampp/htdocs/bizcyclone/files_tenders/'.basename($value['tender_url']); //$destination = '/srv/www/dev.bizcyclone.com/files_tenders/'.basename($source); //$destination = config('app.project.document_file_upload_path').'files_tenders/'.basename($source); $file = fopen($destination,'w+'); fputs($file,$data); $file_mimie_type =mime_content_type($file); fclose($file); if($file_mimie_type=='application/pdf') { $name = 'iadb-'.$value['id'].'-'.time().'-'.uniqid().'-'.str_random(20).'.pdf'; } if($file_mimie_type=='application/msword') { $name = 'iadb-'.$value['id'].'-'.time().'-'.uniqid().'-'.str_random(20).'.doc'; } if($file_mimie_type=='application/zip') { $name = 'iadb-'.$value['id'].'-'.time().'-'.uniqid().'-'.str_random(20).'.zip'; } /*------------ | Update Document and watermark ---------------------*/ if(file_exists('files_tenders/'.basename($value['tender_url']))) { rename('files_tenders/'.basename($value['tender_url']),'files_tenders/'.$name); if($file_mimie_type=='application/pdf'){ $this->LoginUserLib->build_watermark_file($name); } $resp = $this->IadbModel->where('id',$value['id'])->update(['document_name'=>$name,'is_document'=>1]); if($resp){ $process_records++; } } } } catch(\Exception $e){ echo $e->getMessage(); } } } echo"[total_process_records=>".$process_records."]";exit; } else{ echo"No Records Found";exit; } } /*---------- | Translate Tender Title,Company and Description ----------------*/ public function build_language_translation($tender_title) { $directory_arr = []; if(!empty($tender_title)) { try { $ips = $this->proxy_ipaddress(); if($ips!=false) { $tr = new TranslateClient(null, 'en',['timeout' => 1000]); $text = $tr->setHttpOption(['proxy' => $ips])->translate($tender_title); if(!empty($text)) { $tender_title = ""; $directory_arr['tender_title'] = $text; } } } catch(\Exception $e){ echo $e->getMessage(); } } return $directory_arr; } /*---------- | Proxy IPS ------------------*/ public function proxy_ipaddress() { $wrapper_arr =[]; $crawler = $this->client->request('GET', 'https://www.proxynova.com/proxy-server-list/country-in/'); if($crawler->filter('div#bla')->count()>0) { $crawler->filter('div#bla > table > tbody >tr')->each(function($tr_node,$j) use (&$wrapper_arr) { if($tr_node->filter('td')->count()>0) { $port =''; $address =''; $tr_node->filter('td')->each(function($td_node,$i) use(&$wrapper_arr,$j,$address,$port) { if($i==0) { if($td_node->filter('abbr')->count()>0) { $address = $td_node->filter('abbr')->attr('title'); if(preg_match('/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\z/', $address)){ $wrapper_arr[$j]['address']= $address; } else{ $wrapper_arr[$j]['address']= ""; } } else{ $wrapper_arr[$j]['address']= ''; } } if($i==1) { if($td_node->filter('a')->count()>0) { $port = $td_node->filter('a')->html(); $wrapper_arr[$j]['port']= $port; } elseif($td_node->text()!='') { $port = trim($td_node->text()); $wrapper_arr[$j]['port']= $port; } else{ $wrapper_arr[$j]['port']= ''; } } }); } }); if(isset($wrapper_arr) && sizeof($wrapper_arr)>0) { foreach ($wrapper_arr as $key => $row) { if(!empty($row['address']) && !empty($row['port'])) { $ip = $row['address']; $port = $row['port']; $curl = curl_init(); $post_data=$ip.":".$port; return $post_data; } } } else{ return false; } } else{ return false; } } /*------------- | Check Tender Duplication --------------------*/ public function is_tender_exist($original_tender_title) { if(!empty($original_tender_title)) { $obj_tender = $this->IadbModel->where('original_tender_title',$original_tender_title)->count(); if($obj_tender>0){ return false; } else{ return true; } } else{ return false; } } }