The task that was - is to find as much news, RSS feeds and collect them all in one database. When the question arose of how to search news sites, came the idea of using the service news.google.com. From there it is possible to pull out links to news sites, plus they are constantly being added and sorted by region, theme, etc. You're all sections news.google.com, pull links news sites, and then find all the RSS of each.
The code first. It consists of the following files: main.cpp LinksReader.cpp LinksReader.h LinksReader.pro Sources.txt and the file which will be created automatically and save the results in which RssLinks.txt. All the logic is contained in the files and LinksReader.h LinksReader.cpp, other files support, to class and compile the project. Compiled project with a compiler g+ +, Wednesday Qt creator, version 5.1 libraries LinksReader.h
# Ifndef LINKSREADER_H
# define LINKSREADER_H
# Include <QStringList>
<QFile> # include
# include <QNetworkReply>
<QEventLoop> # include
# include <QDebug>
class LinksReader: Public QObject
{
Q_OBJECT
Private :
void loadSources ();
void loadRssLinks ();
void readPage (QString url);
void takeLinks ();
void takeRssLinks ();
void saveRssLinks ();
QNetworkAccessManager mNAManager;
QString mPage;
QString mPageUrl;
QStringList mSources;
QStringList mLinks;
QStringList mRssLinks;
Private Slots:
void onReplyFinished (QNetworkReply pReply *);
Public :
LinksReader (QObject * pParent = 0 );
void Run ();
};
# Endif
LinksReader.cpp
# Include "LinksReader.h"
void LinksReader loadSources :: ()
{
QString fileName = "Sources.txt" ;
QFile file (fileName);
if (! File.Open (QIODevice :: ReadOnly | QIODevice :: Text))
{
qDebug () << "File" << fileName << "not found" ;
return ;
}
QTextStream in (& file);
while (! in.atEnd ())
mSources.push_back (in.readLine ());
file.close ();
qDebug () << mSources.size () << "sources loaded" ;
return ;
}
void LinksReader loadRssLinks :: ()
{
QString fileName = "RssLinks.txt" ;
QFile file (fileName);
if (! File.Open (QIODevice :: ReadOnly | QIODevice :: Text))
return ;
QTextStream in (& file);
while (! in.atEnd ())
mRssLinks.push_back (in.readLine ());
file.close ();
qDebug () << mRssLinks.size () << "Links rss loaded" ;
return ;
}
void LinksReader :: readPage (QString url)
{
mpage = "" ;
mPageUrl = url;
mNAManager.get (QNetworkRequest (QUrl (url)));
QEventLoop loop;
QObject :: connect (& mNAManager, SIGNAL (finished (QNetworkReply *)), & loop, SLOT (quit ()));
loop.exec ();
}
void LinksReader takeLinks :: ()
{
QStringList fullLinks;
QString Links = "" ;
QString beginTN = "<link>" ;
QString endTN = "</ Link>" ;
QString tagContent = "" ;
int beginTP = 0 ;
int endTP = 0 ;
while ( 1 )
{
beginTP = mPage.indexOf (beginTN);
endTP = mPage.indexOf (endTN);
if (beginTP == - 1 | | endTP == - 1 ) Break ;
tagContent = mPage.mid (beginTP + beginTN.length (), endTP - beginTP - endTN.length () + 1 );
links + = tagContent;
mPage.remove ( 0 , endTP + endTN.length ());
}
fullLinks links.split = ( "http://" );
fullLinks.removeFirst ();
for ( int I = 0 ; i <fullLinks.size (); i + +)
{
fullLinks [I]. Remove (fullLinks [I]. indexOf ( "/" ), fullLinks [I]. length ());
mLinks.push_back (fullLinks [i]);
}
}
void LinksReader takeRssLinks :: ()
{
QString beginTN = "<Link" ;
QString endTN = ">" ;
QString tagContent = "" ;
QString beginRssString = "href = \" " ;
QString endRssString = "" ;
int beginTP = 0 ;
int endTP = 0 ;
while ( 1 )
{
beginTP = mPage.indexOf (beginTN);
endTP = mPage.indexOf (endTN, beginTP);
if (beginTP == - 1 | | endTP == - 1 ) Break ;
beginTP + = beginTN.length ();
tagContent = mPage.mid (beginTP, endTP - beginTP);
if (tagContent.indexOf ( "type = \" application / rss + XML \ "" )! = - 1 )
{
int = beginRssPos tagContent.indexOf (beginRssString);
int = endRssPos tagContent.indexOf (endRssString, beginRssPos);
beginRssPos + = beginRssString.size ();
QString rssString = tagContent.mid (beginRssPos, endRssPos - beginRssPos). Remove ( "\" " );
if (rssString.size ()> 0 && rssString [rssString.size () - 1 ] == '/' )
rssString.remove (rssString.size () - 1 , 1 );
if (rssString.indexOf ( "http://" ) == - 1 )
rssString.push_front (mPageUrl);
qDebug () << rssString;
mRssLinks.push_back (rssString);
}
mPage.remove ( 0 , endTP + endTN.length ());
}
}
void LinksReader saveRssLinks :: ()
{
QFile file ( "RssLinks.txt" );
file.open (QFile :: ReadWrite);
QTextStream in (& file);
for ( int I = 0 ; i <mRssLinks.size (); i + +)
<< mRssLinks in [I] << "\ n" ;
file.close ();
}
void LinksReader :: onReplyFinished (QNetworkReply * reply) {
mPage + = reply-> readAll ();
}
LinksReader :: LinksReader (QObject * pParent): QObject (pParent) {
QObject :: Connect (& mNAManager, SIGNAL (finished (QNetworkReply *)), this , SLOT (onReplyFinished (QNetworkReply *)));
}
void LinksReader :: Run ()
{
loadSources ();
loadRssLinks ();
qDebug () << "Please wait ..." ;
for ( int I = 0 ; i <mSources.size (); i + +)
{
readPage (mSources [i]);
takeLinks ();
}
mLinks.removeDuplicates ();
for ( int I = 0 ; i <mLinks.size (); i + +)
{
readPage ( "http://" + MLINKS [I]);
takeRssLinks ();
}
mRssLinks.removeDuplicates ();
saveRssLinks ();
qDebug () << "Finish" ;
}
main.cpp
# Include <QCoreApplication>
# include "LinksReader.h"
int main ( int argc, char * argv [])
{
QCoreApplication a (argc, argv);
LinksReader linksReader;
linksReader.run ();
return a.exec ();
}
LinksReader.pro
QT + = core
QT + = network
QT - = gui
TARGET = LinksReader
CONFIG + = console
CONFIG - = app_bundle
TEMPLATE = app
SOURCES + = main.cpp \
LinksReader.cpp
HEADERS + = \
LinksReader.h
Sources.txt
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
http:
After starting the program, create a class object and invoke its LinksReader method run the following occurs:
1. Loaded sources.
2. Loaded, all found in the last times RSS links (not to search each time a new and replenish an existing database RSS).
3. Go through all the source, read the markings and pull out all the links that are tagged markup <link> </ Link> .
4. Remove all duplicate references.
5. Go through all the news sites found, read the markup of each and pull out the values of all RSS Tag this site.
6. Remove all duplicate RSS links.
7. Finally save the result to a file. Conforming to all RSS links site should be tagged <link rel = "alternate" type = "application / rss + xml" title = "My RSS-feed" href = "index.xml" / > , which is much easier to find RSS. As can be seen the program is simple enough, but does its job well. One run is 600-700 RSS feeds. Restarting increase this amount, so as to news.google.com constantly added new links to news sites. did not want to consider in detail each line of code, I think that he was so well read, not even knowing Qt. But if there is a strange place - ask and I will gladly help you to understand. github
No comments:
Post a Comment