[es] - Extracting data from HTML

^{07.07.2013. u 08:34 - pre 145 meseci}

Oglas je na engleskom (copy pasted), korespodencija medjutim moze i na srpskom :).

We need a number of rather simple HTML parsers to be written in C#. Their purpose is to extract useful information from various airline website, respecting the websites' terms and conditions. Each parser must be a class inherited from our base class. Each class is almost never longer than 250 lines of quality code. The list of actual websites to be parsed, necessary base code, detailed step-by-step instructions as well as a couple of examples are subsequently provided.

Experience in .net framework, htt(p) protocol [get, post, cookies] and regular expressions is essential. You must be able to write clean efficient code. Interest in airlines or online travel industry is a plus.

Work from home and write as many classes as you wish, choosing any from the provided list. Payment is by Paypal or Skrill and starts at 90 euro per each submitted class. Cashing in Belgrade is possible but a 15 euro fee applies. Repeated submissions of quality code quickly lead to higher payment rates.

Applications should be sent to [email protected]. Please attach your CV. Any questions can be posted in this thread.

A simple example is provided below.

Code:

using System;
using System.Collections.Generic;
using System.Text;
using System.Collections.Specialized;
using System.Globalization;
using System.Text.RegularExpressions;

namespace Azuon
{
    class AirArabia : Airline
    {
        protected override string dataStr
        {
            get
            {
                return
@"http://

GetCookie
HEAD www.airarabia.com/home

[no]

[no]

SearchFare
POST reservationsma.airarabia.com/ibe/public/showReservation.action

[no]

hdnCarrier
hdnDesign    0
hdnLanguage
hdnParamUrl

GetJSON
POST reservationsma.airarabia.com/ibe/public/availabilitySearchCalendar.action?requestID=1

[no]

fareQuoteJson
flexiSelection
searchParams.departureVariance    3
searchParams.returnVariance    3
searchParams.selectedCurrency    EUR
searchParams.fareType    A
searchParams.searchSystem    INT
searchParams.classOfService    Y
searchParams.firstDeparture
searchParams.lastArrival
selectedFlightJson
searchParams.halfReturnFareQuote
searchParams.stayOverTimeInMillis
searchParams.fromAirportSubStation
searchParams.toAirportSubStation
searchParams.fareQuoteLogicalCCSelection
searchParams.quoteOutboundFlexi    true
searchParams.quoteInboundFlexi    true
searchParams.flexiSelected
";
            }
        }

        public AirArabia()
        {
            name = "Air Arabia";
            code = "G9";
            LUGGAGE_FEE = 0;
            DOUBLE_BOOKING_FEE_FOR_ROUND_TRIP = false;
            paymentMethods = new Dictionary<string, float> {
                { "Visa", 0 },
                { "Mastercard", 0}};
        }

        protected override NameValueCollection getSearchParameters(Query query)
        {
            NameValueCollection pars = new NameValueCollection();
            pars.Add("hdnParamData",
                "EN^AS^" +
                query.origin.code + "^" +
                query.destination.code + "^" +
                query.dates[0].ToString("dd/MM/yyyy", CultureInfo.InvariantCulture) + "^" +
                (query.isRound() ? query.dates[1].ToString("dd/MM/yyyy", CultureInfo.InvariantCulture) : "") + "^" +
                "0^0^EUR^" +
                query.passgs.adults.ToString() + "^" +
                query.passgs.children.ToString() + "^" +
                query.passgs.infants.ToString() + "^" +
                "FROM^TO^" +
                query.isRound().ToString().ToLower());
            return pars;
        }

        protected override bool parseFares(Query query, string html, List<Fare>[] foundFares)
        {
            var pars = new NameValueCollection() {
                { "searchParams.adultCount", query.passgs.adults.ToString() },
                { "searchParams.childCount", query.passgs.children.ToString() },
                { "searchParams.infantCount", query.passgs.infants.ToString() },
                { "searchParams.fromAirport", query.origin.code },
                { "searchParams.toAirport", query.destination.code},
                { "searchParams.departureDate", query.dates[0].ToString("dd/MM/yyyy", CultureInfo.InvariantCulture)},
                { "searchParams.returnDate", query.isRound() ? query.dates[1].ToString("dd/MM/yyyy", CultureInfo.InvariantCulture) : "" },
                { "searchParams.returnFlag", query.isRound().ToString() },
                { "searchParams.fromAirportName", query.origin.name },
                { "searchParams.toAirportName", query.destination.name }
            };

            var json = executeCommand("GetJSON", pars, true);

            var mFare = new Regex(
                "\"calDisplayAmout\":(?<price>[0-9]+)," +
                ".+?" +
                "\"returnFlag\":(?<dir>true|false)," +
                "\"segments\":\\[(?<legs>[^]]+)\\]", RegexOptions.Singleline).Match(json);

            if (!mFare.Success)
            {
                return false;
            }

            while (mFare.Success)
            {
                var isOutbound = mFare.Groups["dir"].Value == "false";
                var fare = new Fare(query, isOutbound);

                const int INFANT_FEE = 30;

                fare.price = Convert.ToSingle(mFare.Groups["price"].Value) * query.passgs.Count() + INFANT_FEE * query.passgs.infants;

                var sLegs = mFare.Groups["legs"].Value;

                // take departure of 1st leg
                var depUnixSeconds = new Regex(
                    "\"departureTimeLong\":(?<num>[0-9]+),").Match(sLegs);

                fare.departTime = Utils.unixStart.AddSeconds(Convert.ToInt64(depUnixSeconds.Groups["num"].Value) / 1000);

                // take arrival of last leg
                var arrUnixSeconds = new Regex(
                    "\"arrivalTimeLong\":(?<num>[0-9]+),", RegexOptions.RightToLeft).Match(sLegs);

                fare.arriveTime = Utils.unixStart.AddSeconds(Convert.ToInt64(arrUnixSeconds.Groups["num"].Value) / 1000);

                var mVia = new Regex(
                    "\"segmentShortCode\":\"\\w\\w\\w[^A-Z]+(?<via>\\w\\w\\w)\"").Match(sLegs);

                var sVia = mVia.Groups["via"].Value;

                if ((isOutbound && sVia != query.destination.code) ||
                    (!isOutbound && sVia != query.origin.code))
                {
                    fare.setVia(sVia);
                }

                foundFares[Convert.ToByte(!isOutbound)].Add(fare);

                mFare = mFare.NextMatch();
            }

            return true;
        }
    }