#96 Update homepage section around centos
Merged 3 years ago by arrfab. Opened 3 years ago by areguera.
centos/ areguera/centos.org master  into  master

file added
+1641
The added file is too large to be shown here, see it at: _data/centos/planet.yml
file modified
+7 -1
@@ -1,2 +1,8 @@ 

+ 

  <h2>Around CentOS</h2>

- <div id="rss-feeds"></div>

+ <ul>

+ {% assign channels = site.data.centos.planet.rss.channel %}

+ {% for i in channels limit:12 %}{% if forloop.index > 4 %}

+ <li><a href="{{ i.item.link }}">{{ i.item.title }}</a><br>{{ i.item.description | strip_html | slice: 0,150 }}...</li>

+ {% endif %}{% endfor %}

+ </ul>

file modified
-12
@@ -23,18 +23,6 @@ 

    {% include footer.html -%}

    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>

    <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.4.1/js/bootstrap.bundle.min.js"></script>

-   <script src="https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.8.4/moment.min.js"></script>

-   <script src="/assets/js/jquery.rss.js"></script>

-   <script>

-     jQuery(function($) {

-       $("#rss-feeds").rss("https://www.centos.org/assets/planet.rss", {

-         limit: 5,

-         ssl: true,

-         entryTemplate: '<li><a href="{url}">{date} : {title}</a><br/>{shortBodyPlain}</li>'

-         },

-       )

-    })

-   </script>

  </body>

  

  </html>

file removed
-11
@@ -1,11 +0,0 @@ 

- (function(d){var e=function(a,b,c,f){this.target=a;this.url=b;this.html=[];this.effectQueue=[];this.options=d.extend({ssl:!1,host:"www.feedrapp.info",limit:null,key:null,layoutTemplate:"<ul>{entries}</ul>",entryTemplate:'<li><a href="{url}">[{author}@{date}] {title}</a><br/>{shortBodyPlain}</li>',tokens:{},outputMode:"json",dateFormat:"dddd MMM Do",dateLocale:"en",effect:"show",offsetStart:!1,offsetEnd:!1,error:function(){console.log("jQuery RSS: url doesn't link to RSS-Feed")},onData:function(){},

- success:function(){}},c||{});this.callback=f||this.options.success};e.htmlTags="doctype,html,head,title,base,link,meta,style,script,noscript,body,article,nav,aside,section,header,footer,h1-h6,hgroup,address,p,hr,pre,blockquote,ol,ul,li,dl,dt,dd,figure,figcaption,div,table,caption,thead,tbody,tfoot,tr,th,td,col,colgroup,form,fieldset,legend,label,input,button,select,datalist,optgroup,option,textarea,keygen,output,progress,meter,details,summary,command,menu,del,ins,img,iframe,embed,object,param,video,audio,source,canvas,track,map,area,a,em,strong,i,b,u,s,small,abbr,q,cite,dfn,sub,sup,time,code,kbd,samp,var,mark,bdi,bdo,ruby,rt,rp,span,br,wbr".split(",");

- e.prototype.load=function(a){var b="http"+(this.options.ssl?"s":"")+"://"+this.options.host+"?callback=?&q="+encodeURIComponent(this.url);this.options.offsetStart&&this.options.offsetEnd&&(this.options.limit=this.options.offsetEnd);null!==this.options.limit&&(b+="&num="+this.options.limit);null!==this.options.key&&(b+="&key="+this.options.key);d.getJSON(b,a)};e.prototype.render=function(){var a=this;this.load(function(b){try{a.feed=b.responseData.feed,a.entries=b.responseData.feed.entries}catch(c){return a.entries=

- [],a.feed=null,a.options.error.call(a)}b=a.generateHTMLForEntries();a.target.append(b.layout);if(0!==b.entries.length){d.isFunction(a.options.onData)&&a.options.onData.call(a);var f=d(b.layout).is("entries")?b.layout:d("entries",b.layout);a.appendEntriesAndApplyEffects(f,b.entries)}0<a.effectQueue.length?a.executeEffectQueue(a.callback):d.isFunction(a.callback)&&a.callback.call(a)})};e.prototype.appendEntriesAndApplyEffects=function(a,b){var c=this;d.each(b,function(b,e){var d=c.wrapContent(e);"show"===

- c.options.effect?a.before(d):(d.css({display:"none"}),a.before(d),c.applyEffect(d,c.options.effect))});a.remove()};e.prototype.generateHTMLForEntries=function(){var a=this,b={entries:[],layout:null};d(this.entries).each(function(){var c=a.options.offsetStart,f=a.options.offsetEnd;c&&f?index>=c&&index<=f&&a.isRelevant(this,b.entries)&&(c=a.evaluateStringForEntry(a.options.entryTemplate,this),b.entries.push(c)):a.isRelevant(this,b.entries)&&(c=a.evaluateStringForEntry(a.options.entryTemplate,this),

- b.entries.push(c))});b.layout=this.options.entryTemplate?this.wrapContent(this.options.layoutTemplate.replace("{entries}","<entries></entries>")):this.wrapContent("<div><entries></entries></div>");return b};e.prototype.wrapContent=function(a){return 0!==d.trim(a).indexOf("<")?d("<div>"+a+"</div>"):d(a)};e.prototype.applyEffect=function(a,b,c){switch(b){case "slide":a.slideDown("slow",c);break;case "slideFast":a.slideDown(c);break;case "slideSynced":this.effectQueue.push({element:a,effect:"slide"});

- break;case "slideFastSynced":this.effectQueue.push({element:a,effect:"slideFast"})}};e.prototype.executeEffectQueue=function(a){var b=this;this.effectQueue.reverse();var c=function(){var f=b.effectQueue.pop();f?b.applyEffect(f.element,f.effect,c):a&&a()};c()};e.prototype.evaluateStringForEntry=function(a,b){var c=a,f=this;d(a.match(/(\{.*?\})/g)).each(function(){var a=this.toString();c=c.replace(a,f.getValueForToken(a,b))});return c};e.prototype.isRelevant=function(a,b){var c=this.getTokenMap(a);

- return this.options.filter?this.options.filterLimit&&this.options.filterLimit===b.length?!1:this.options.filter(a,c):!0};e.prototype.getFormattedDate=function(a){if(this.options.dateFormatFunction)return this.options.dateFormatFunction(a);return"undefined"!==typeof moment?(a=moment(new Date(a)),a=a.locale?a.locale(this.options.dateLocale):a.lang(this.options.dateLocale),a.format(this.options.dateFormat)):a};e.prototype.getTokenMap=function(a){if(!this.feedTokens){var b=JSON.parse(JSON.stringify(this.feed));

- delete b.entries;this.feedTokens=b}return d.extend({feed:this.feedTokens,url:a.link,author:a.author,date:this.getFormattedDate(a.publishedDate),title:a.title,body:a.content,shortBody:a.contentSnippet,bodyPlain:function(a){for(var a=a.content.replace(/<script[\\r\\\s\S]*<\/script>/mgi,"").replace(/<\/?[^>]+>/gi,""),b=0;b<e.htmlTags.length;b++)a=a.replace(RegExp("<"+e.htmlTags[b],"gi"),"");return a}(a),shortBodyPlain:a.contentSnippet.replace(/<\/?[^>]+>/gi,""),index:d.inArray(a,this.entries),totalEntries:this.entries.length,

- teaserImage:function(a){try{return a.content.match(/(<img.*?>)/gi)[0]}catch(b){return""}}(a),teaserImageUrl:function(a){try{return a.content.match(/(<img.*?>)/gi)[0].match(/src="(.*?)"/)[1]}catch(b){return""}}(a)},this.options.tokens)};e.prototype.getValueForToken=function(a,b){var c=this.getTokenMap(b),d=a.replace(/[\{\}]/g,""),d=c[d];if("undefined"!==typeof d)return"function"===typeof d?d(b,c):d;throw Error("Unknown token: "+a+", url:"+this.url);};d.fn.rss=function(a,b,c){(new e(this,a,b,c)).render();

- return this}})(jQuery);

  • The "Around CentOS" section in the homepage was built using the
    jquery.rss.js. Recently we detected the content was not any longer
    visible and checking the script found it was consuming an external
    site (http://www.feedrapp.info/) that is currently out of service. As
    consequence the "Around CentOS" section in the homepage shows no
    information.

This update converts the file "assets/planet.rss" into
"_data/centos/planet.yml" and updates the section code to read this
file instead. The jquery.rss.js file was removed and the approach it
provides no longer considered. So to prevent this from happening
again.

A future commit must remove the planet.yml provided in this commit, so
another one, automatically generated take its place. This way updates
from planet can be visible in the home page.

For the RSS to YAML conversion used the following python script:


import xmlplain

# Read to plain object
with open("assets/planet.rss") as inf:
root = xmlplain.xml_to_obj(inf, strip_space=True, fold_dict=True)

# Output plain YAML
with open("_data/centos/planet.yml", "w") as outf:
xmlplain.obj_to_yaml(root, outf)


Hmm, does that mean that it wouldn't fetch "on demand" (client side) the new needed rss feeds to be displayed in browser but instead would rely on external script/cron generating planet.yml ?
And is then planet.yml on website is enough is does it require jekyll to process that external file (not in git) to build/refresh website ?

Hmm, does that mean that it wouldn't fetch "on demand" (client side) the new needed rss feeds to be displayed in browser but instead would rely on external script/cron generating planet.yml ?

Yes. There wouldn't be any fetch action from the client side. Everything would be based on the "_site/assets/planet.rss" file and the script used to update this file. In this point I am considering there is already a script somewhere to update the "_site/assets/planet.rss" file, which is the input file provided to jquery.rss.js script.

For example, would it be possible to add the xml-to-yaml conversion block of code to the script already used to update the "_site/assets/planet.rss" file? So, once the "_site/assets/planet.rss" file gets in place its content be immediately transformed into "_data/centos/planet.yml", as part of the same update process, no extra script/cronjob running on a separate moment of time.

And is then planet.yml on website is enough is does it require jekyll to process that external file (not in git) to build/refresh website ?

Yes. It would be similar to the process we already have in place for download mirrors .csv files. Once the "_data/centos/planet.yml" file is in place a site rebuild is necessary to render the section with the new information the file provides. To attend this we may use a "jekyll build --watch" running process on the background so, changes can appear in an unattended fashion, once the .yml file changes. Note that by using the --watch option, jekyll only rebuilds the files that change, it doesn't rebuild the the entire site.

In case it helps, here it is a tentative README.md update with documentation about running jekyll build --watch inside a container controlled by systemd.

Cool, I tested locally this PR and I'll merge it.
But new push should be removing the _data/centos/planet.yml file and have it generated on periodic refresh but not through git : I'll plumb it in the existing script that indeed download validated/current mirrors before rebuilding and pushing refreshed website.

Thanks a lot @areguera !

Pull-Request has been merged by arrfab

3 years ago

@areguera : tried to find which rpm pkg would provide xmlplain python module but couln't find one easily , but I'll just modify the script that download rss on node building website once I'll have tested and found one that can parse xml and render yaml

For xmlplain I used pip3 install --user xmlplain. Could this be an option?

There is something to consider about the rss to yaml conversion and it is the order of the first 4 items inside the channel dictionary the rss file provides. If that order changes in the conversion the result shown in the web page may be one different from the one expected.

Correction: Inside the YAML file, channel is a list not a dictionary. The dictionary are the fifth and following elements of the list. All named item. The YAML structure may look as the following:

rss:
  channel:
  - title: Planet CentOS
  - link: https://planet.centos.org
  - language: en
  - description: Planet CentOS - https://planet.centos.org 
  - item:
      title: ""
      guid: ""
      link: ""
      description: ""
      pubDate: ""
  - item:
      title: ""
      guid: ""
      link: ""
      description: ""
      pubDate: ""

For jekyll to show the content as expected, the code put in the "Around CentOS" section iterates in the channel list, excludes the first 4 elements from being processed and works with the rest of item dictionaries.

we prefer no depending on pip on infra fleet and rather have a rpm, so if there is no other python xml parser (there are quite some normally) that produces the same result, we'll have to see about a .spec for rpm pkg to be built through infra tag

python3-feedparser (available in epel8 and so easy to retag for infra tag) seems a good candidate to parse rss feed and then reconstruct a yaml ?

Yes it does :) I am going to look at it this weekend to find a way through it. I see its default output is in json format which jekyll should understand as well. In this case, references in jekyll includes may need to be updated in order to find the specific entries we want to show.

The yaml file reconstruction is indeed an option too.

I opened PR#99 to continue this.