Using http://www.dhtmlx.com/ widgets to display ActiveMQ's demo/portfolio.html

Dhtml Table

Dhtml Table and Graphs

Data conversion

Some data conversion

import time

dayNames = {0:'Mon',1:'Tue',2:'Wed',3:'Thu', 4:'Fri',5:'Sat',6:'Sun'}
monthNames = {1:'Jan',2:'Feb',3:'Mar',4:'Apr',5:'May',6:'Jun',7:'Jul',8:'Aug',9:'Sep',10:'Oct',11:'Nov',12:'Dec'}

nameToNumber = {}
lines = []
with open("C:\RawCallLogs.csv","r") as f:
    for line in f:
        fields = line.split(',')
        folder = fields[0].replace('\"','')
        if folder == 'Folder':
            continue
        number = fields[1].replace('\"','')
        name = fields[2].replace('\"','')
        rawtime = fields[3].replace('\"','')
        structTime = time.strptime(rawtime,'%m/%d/%Y %H:%M:%S')
        weekday = "Weekday"
        if (structTime.tm_wday >= 5):
            weekday = "Weekend"
        dayName = dayNames[structTime.tm_wday]
        month = monthNames[structTime.tm_mon]
        day = structTime.tm_mday
        year = structTime.tm_year
        evening = "Daytime"
        if structTime.tm_hour > 18:
            evening = "Evening"
        duration = int(fields[4].replace('\"','').replace('\n',''))/60.0
        if number == '-1':
            name = 'Unknown'

        if name != "":
            if name not in nameToNumber:
                nameToNumber[name]  = set([number])
            else:
                nameToNumber[name].add(number)
        lines.append([folder,number,name,rawtime,duration,weekday,evening,dayName,month,day,year,structTime.tm_hour,structTime.tm_min])
        
print len(lines)

def is08Number(number):
    return number.startswith('08') or number.startswith('0448')

def isLandlineNumber(number):
    return number.startswith('02') or number.startswith('0442')

def isMobile(number):
    return len(number) > 7

def isInternational(number):
    return number.startswith('0091') or number.startswith('+91')

with open("C:\RawCallLogsOut.csv","w") as f:
    f.write('Folder,Number,Name,Raw time,Duration,Weekday(s),DayTime,DayofWeek,Month,Day,Year,Hour,Min,NumberType\n')

    for line in lines:
        if (line[2] == ''):# No name
            for name,numbers in nameToNumber.items():
                for number in numbers:
                    numberInLine = line[1][::-1][:6][::-1]
                    numberInMap = number[::-1][:6][::-1]
                    if numberInLine == numberInMap:
                        line[2] = name
                        break
        if (line[2] == ''):
            if is08Number(line[1]):
                line[2] = "Unknown 08 Number"
            elif isLandlineNumber(line[1]):
                line[2] = 'Unknown landline'
            elif isMobile(line[1]):
                line[2] = 'Unknown mobile'
            else:
                line[2] = 'Unknown'
        numberType = 'Unknown'
        if (is08Number(line[1])):
            numberType = "08 Number"
        elif isLandlineNumber(line[1]):
            numberType = "Landline"
        elif isInternational(line[1]):
            numberType = "International"
        elif isMobile(line[1]):
            numberType = "Mobile"
        f.write("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n" % (line[0],line[1],line[2],line[3],line[4],line[5],line[6],line[7],\
                                                     line[8],line[9],line[10],line[11],line[12],numberType))

Python: Accessing https pages from behing an authenticating proxy server

from urllib2 import ProxyHandler, build_opener

proxyMap = {'https': 'http://(user):(pass)@(proxyhost):(proxypass)/'}
proxy = ProxyHandler(proxyMap)
opener = build_opener(proxy)
u = opener.open('https://somehttpspage')
data = u.read()



charset = u.info().getparam('charset') # Python 2
print 'CharSet %s' % charset
print '------------------------------------------------------------------------'
print data

Python script for creating video using images and adding audio

I am usng a python script for converting images into a vidoe (using Java JMF) and then adding the audio to the vidoe and converting it to an mp4 format so that I can play it on my phone.

'''
Created on 15 Apr 2010

@author: viswanav
'''
import glob, subprocess, os

vlcLocation = "C:\Development\Apps\PortableApps\VLCPortable\App\\vlc\\vlc.exe"

for g in glob.glob("C:\Development\Docs\Books\VideoCDs\*\PK*"):
    audioFile = ""
    for flvGlob in glob.glob(g + "\*\media\*.flv"):
        audioFile = flvGlob
        break;
    
    
    convertToMov = ["java","-cp","C:\Development\workspaces\Trials\JavaTrials\JMF\out\production\JMFTrial","JpegImagesToMovie", "-w", "400","-h","300","-f","15",g]
    #print convertToMov
    print "Convert to Mov File"
    subprocess.call(convertToMov)
    
    movFileLocation = ""
    for movFile in glob.glob(g + "\*.mov"):
        movFileLocation= movFile
        break
    
    stdoutCommand = "--sout=#transcode{vcodec=mp4v,vb=800,acodec=mp4a,ab=128,scale=1,channels=2,audio-sync}:std{access=file,mux=mp4,dst=\"%s.mp4\"}" % movFileLocation
    print stdoutCommand  
    addAudioAndConverToMp4Command = [vlcLocation,"-I","dummy",movFileLocation,"--input-slave=\"%s\"" % audioFile,stdoutCommand,"vlc://quit"]
    print addAudioAndConverToMp4Command 
    print "Creating mp4 file"
    subprocess.call(addAudioAndConverToMp4Command)
    
    deleteMovCommand = ["del",movFileLocation]
    print deleteMovCommand
    print "Deleting mov file"
    os.remove(movFileLocation)

Python application for downloading a book and converting to pdf

A similar application in python for downloading book images from the net and converting it to a pdf file, but this time using python

'''
Created on 25 Mar 2010


'''
import httplib
import os
import sys
from optparse import OptionParser

import time, thread
from reportlab.pdfgen import canvas
from reportlab.lib.pagesizes import letter, A4
from reportlab.lib.units import cm, mm, inch, pica

write = sys.stdout.write
h = httplib.HTTPConnection('webproxy',8080)
headers = {'Cookie':''}
size = 1800
urltemplate = '/%s/pages/%i.jpg?width=1800'
books = ('isbn2','isbn1')
maxPages = 10000

def downloadBookImages(folderLocation,isbn):
 bookLocation = folderLocation + "\\%s" % isbn
 print 'Downloading book to %s' % bookLocation
 #Check if the folder location exists
 if os.path.exists(bookLocation) == False:
  os.mkdir(bookLocation) # If not create the folder
 h = httplib.HTTPConnection('webproxy',8080)
 
 imageFilePathTemplate = bookLocation + "\\%i.jpg"
 # Loop through all the pages for the book and save the page
 for page in range(maxPages):
  # Create the image file path
  imageFilePath = imageFilePathTemplate % page;
  # Check if the file exists
  if os.path.exists(imageFilePath) == True:
   write('E%i ' % page)  
   continue
  url = urltemplate % (isbn,page)
  h.request('GET', url,headers=headers)
  write('.')
  responce = h.getresponse()
  if responce.status == 401:
   print 'Not Authorized Book %s, the cookie has expired' % isbn
   break;
  data = responce.read()
  if len(data) < 1000:
   print 'End of book %s since the page size is too small : %i' % (isbn,len(data))
   break
  f = open(imageFilePath,'wb')
  f.write(data)
  f.close()


def pdfDirectory(outputPDFName, imageDirectory ):
    dirim = str(imageDirectory)
    output = str(outputPDFName)
    print 'Converting to pdf %s, images %s' % (output,dirim)
    width, height = A4
    height, width = A4
    c = canvas.Canvas(output, pagesize=A4)
    for root, dirs, files in os.walk(dirim):
     nopage = len(files)
     for name in range(nopage):
      #print name/nopage
      write('.')
      name = str(name) + ".jpg"
      filepath = os.path.join(root, name)
      #if filepath.endswith('20.jpg'): break
      c.drawImage(filepath, mm * 0.001 , mm * 0.001, height, width, preserveAspectRatio=False)
      ##c.showPage()
      c.save()
    print "PDF of Image directory created %s" % outputPDFName


def threadExe(location,isbn):
 print 'Executing Thread for location %s book %s\n' % (location,isbn)
 for page in range(maxPages):
  print '%s Page - %i \n' % (isbn,page)
  time.sleep(10)


def createOptionParser():
 parser = OptionParser(usage="Usage: %prog [options]", version="%prog 1.0")
 parser.add_option("-i", "--isbn", dest="isbn", action='store', type='string')
 parser.add_option("-d", action="store_true", dest="download", help="Download the book images", default=False)
 parser.add_option("-c", action="store_true", dest="converToPdf", help="Convert images to pdf file", default=False)
 parser.add_option("-f", "--imageFolder", type="string", help="Image folder to download or to read from.")
 parser.add_option("-p", "--pdfOutputFolder", type="string", help="Pdf output folder.")
 return parser

if __name__ == '__main__':
 parser = createOptionParser()
 (options, args) = parser.parse_args()
 if not options.isbn:
  parser.error("You have to specify the isbn book number")
 
 if options.download or options.converToPdf:
  if not options.imageFolder:
   parser.error("Specify the image folder")
  
  if options.download:
   downloadBookImages(options.imageFolder,options.isbn)
   
  if options.converToPdf:
   if not options.pdfOutputFolder:
    parser.error("Specify the pdf folder")
   print 'Converting to pdf %s, images %s' % (options.pdfOutputFolder,options.imageFolder)
   pdfDirectory(options.pdfOutputFolder + options.isbn + ".pdf" , options.imageFolder)
 else:
  print 'Specify if you would like to download or convert the book or both?'
   
# 
# for book in books:
#  downloadBookImages('c:\\tmp',book)
# 
# for book in books:
#  pdfDirectory('w:\\%s' % book , 'c:\\tmp\\%s.pdf' % book)
# 

Extracting the audio from an flv file and creating a new flv with only audio

As a part of my studies I wanted to extract out only the audio from the flv file into a new flv file.

Here is what I did

$ffmpeg -i input.flv -f flv -vn -acodec copy output.flv

I also wanted to convert all the files while taking a copy of the original file

$find . -name *.flv -exec cp '{}' '{}'.`date +%d.%m.%y`.original \; -exec ffmpeg -i '{}' -y -f flv -vn -acodec copy '{}' \;

The above command didnt exactly work since, it was overwriting the old file as it was being converted, so I used this one

$find . -name *.flv -exec cp '{}' '{}'.original \; -exec ffmpeg -i '{}'.original -y -f flv -vn -acodec copy '{}' \;

ffmpeg man pages: http://swoolley.org/man.cgi/1/ffmpeg

Mobile Application idea

I am thinking of writing a mobile application for Android, now for creating any application I could go down the route of reading through the Android Dev Guide and end up creating a simple application like notepad or something more complex like a home webcam viewer. The lack of an application idea makes researching into learning about the Android less motivating, as a result of which I give up reading through the Dev Guide half way through.
What I really need is an idea for a application, this application should be unique and it should have a wow factor, it should be something that no one has thought about and it should be generic so that more people can use it, rather than having something very specific to I want. Something on the lines of the mobile based web cam viewer application that I had thought and developed in June 2004.

In order to develop this idea I wanted to write down all the features provided by my Google Phone, but then I thought for developing idea I should start with all the features provided by the platform, but think out of box about the problem that I have solve without being constrained by the set of features provided by the phone.

Lets see how things go, I have to rush for a meeting now

Adding Syntax Highlighting to Blogger

A good description for adding syntax highlighting to a blog post.

http://heisencoder.net/2009/01/adding-syntax-highlighting-to-blogger.html

Extracting and creating a book from an online site

I have a subscription to an online book store (similar to safari). Unfortunately I cannot provide you with the store name. I wanted certain books from the store on my local machine as a printable pdf file. The bookstore allowed me to print only 2 pages.

Here is what I did to extract the data from the book store.
1. Figured out the data format for each page. Each page was a jpg file
2. Figured out the http request for each page. The request contained the ISBN nuber the page number and the resolution
3. Using httpfox - a firefox plugin to introspect the http data sent accross.
4. Wrote this application to fetch all the pages for a set of books and save the image on the disk additionally it also creates the pdf file with all the images.

Rather than using a java application I could have used some kinda scripting language. I may research into that later on if I get the time, as of now I can print a few pages that I want to read :-)

Here is the java source code for Main.java

package bookextracter;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;

import javax.imageio.stream.FileImageOutputStream;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.itextpdf.text.BadElementException;
import com.itextpdf.text.Document;
import com.itextpdf.text.DocumentException;
import com.itextpdf.text.Image;
import com.itextpdf.text.PageSize;
import com.itextpdf.text.pdf.PdfWriter;

/**
 * Library Dependencies
 *   iText (http://itextpdf.com/index.php)
 *   Used for creating pdf documents.
 *   Some really good example code using iText (http://www.roseindia.net/java/itext/index.shtml)
 * 
 *   Apache Commons Logger (http://commons.apache.org/logging/) 
 *   A generic interface for logging.
 * 
 * The high level steps are as follows
 * 1. Log into the online book site using your browser
 * 2. Use an HTTP monitor (something like httpfox for firefox) to introspect the request/response messages
 * 3. As your are browsing through the online book pages note the cookie headers in the GET request for each page. Note down this cookie string 
 * since you will be using it in this program
 * 
 * 
 */
public class Main {

 static Log _logger = LogFactory.getLog(Main.class);
 
 /**
  * The cookie that is unique to my login.
  */
 public static final String cookie = "not real cookie";
 
 /**
  * Each pages is a jpg image and this determines the scaling factor when we make a request from the server. 
  */
 public static final int MAX_SCALE = 1200;
 
 /**
  * I loop through these many number of pages till reach the end or the server responses with code '500'
  * Here I assume that all the books contain a max of 1500 pages.
  */
 public static final int MAX_ESTIMATE_BOOK_SIZE_IN_PGS = 1500;
 
 public static final String bookSaveLocation = "C:\\Extract\\Books\\";
 public static final String rawImageSaveLocation = "C:\\Extract\\RawImages\\";

 public static void main(String[] args) throws HttpException, IOException,
   DocumentException, InterruptedException {

  
  // These are the ISBN numbers of the books that I am interested in.(not real)
  String[] bookIsbs = { "234234234", "1234234234", "234234234"};
  
  // loop through each book and fetch all the pages for each book in a different thread.
  for (String isbn : bookIsbs) {
   (new Thread(new BookFetcher(isbn))).start();
  }
  
  // Wait for it.
  System.in.read();

 }

 private static class BookFetcher implements Runnable {
  private String isbn;

  public BookFetcher(String isbn) {
   this.isbn = isbn;
  }

  @Override
  public void run() {
   try {
    extractAndSaveBook(isbn);
   } catch (Exception e) {
    _logger.error(e);
    e.printStackTrace();
   }

  }
 }

 private static void extractAndSaveBook(String isbn) throws IOException,
   HttpException, FileNotFoundException, DocumentException {
  
  // The format of the pdf doc.
  Document pdfDoc = new Document(PageSize.A2, 0, 0, 0, 0);
//  Document pdfDoc = new Document(PageSize.LETTER,0,0,0,0);
//  Document pdfDoc = new Document();  
  
  // The file name and location of the pdf file 
  final String pdfFileName = bookSaveLocation + isbn + ".pdf";
  PdfWriter.getInstance(pdfDoc, new FileOutputStream(pdfFileName));
  pdfDoc.open();

  // Set up the http Connection
  _logger.debug("Creating an Http Client");
  HttpClient httpClient = new HttpClient();
  httpClient.getHostConfiguration().setProxy("webproxy", 80); // I use a proxy.

  _logger.info("Retriving Book " + isbn);
  
  // Loop through each page.
  for (int currentPageNumber = 0; currentPageNumber < MAX_ESTIMATE_BOOK_SIZE_IN_PGS; currentPageNumber++) {
   
   // Create the page url
   String pageUrl = construcPageUrl(isbn, currentPageNumber, MAX_SCALE);
   _logger.debug("Retrive Page: " + pageUrl);
   
   // Print the page number do indicate progress.
   System.out.print(currentPageNumber + "..");
   if ((currentPageNumber + 1) % 30 == 0)
    System.out.println("");

   // The retrived image for each page will be saved at the following location (file format: <rawImageSaveLocation>/<isbn>/Page_<currentPageNo>.jpg 
   String pagePathName = rawImageSaveLocation + isbn + "\\Page_" + currentPageNumber
     + ".jpg";
   
   File imageFile = new File(pagePathName);
   // If the file dosent exist at the location the fetch the file from the server.
   // This prevents us from hitting the server during multiple runs, since fetching 
   // each page is the most time consuming operation.
   if (!(imageFile.exists() && imageFile.isFile())) { 
    GetMethod getPage = new GetMethod(pageUrl);
    getPage.setRequestHeader("Cookie", cookie); // Setup the http GET request with my cookie 

    int responseCode = httpClient.executeMethod(getPage);
    if (responseCode != 200) {
     _logger.info("Page Not found, I assume we are done with the book");
     break;
    }

    // Get the data
    _logger.debug("Get the Page");
    byte[] pageRawByteArray = getPage.getResponseBody();
    
    // Save the Raw Image to file
    saveRawImageToFile(pagePathName, pageRawByteArray);
   }
   // Save the image page to pdf.
   savePageImageToPdfDoc(pdfDoc, pdfFileName,
     pagePathName);
  }
  System.out.println("");
  _logger.info("Book " + isbn + " ....Done");
  pdfDoc.close();

 }

 private static void savePageImageToPdfDoc(Document pdfDoc,
   final String pdfFileName, String pagePathName)
   throws BadElementException, MalformedURLException, IOException,
   DocumentException {
  _logger.debug("Saving to pdf Doc " + pdfFileName);
  // Image pagePdfImage = Image.getInstance(pageRawByteArray);
  Image pagePdfImage = Image.getInstance(pagePathName);
  // pagePdfImage.scalePercent(50);
  pdfDoc.add(pagePdfImage);
 }

 private static void saveRawImageToFile(String pagePathName,
   byte[] pageRawByteArray) throws FileNotFoundException, IOException {
  _logger.debug("Saving Image to a file : " + pagePathName);
  FileImageOutputStream imageWriter = new FileImageOutputStream(
    new File(pagePathName));
  imageWriter.write(pageRawByteArray);
  imageWriter.close();
 }

 private static String construcPageUrl(String bookIsbn, int pageNo, int scale) {
  return "http://booksitehostname/" + bookIsbn + "
    + scale + "/" + pageNo + ".jpg";
 }

}

Ideas

Some web applications that I would like to see.

Adding notes to sections of a webpage.
I need a web application (with a firefox plugin) that allows me to add notes to sections of a web page. Something similar to selecting sections of a web page and highlighting it, similar to using a highlighter on a book and then writing some notes on the side. I tried searching for something similar on the net, but all that I could get was MyStickies, it allows me to add postit notes to the entire webpage and not to sections of a web page.

I would like to catalog all my sms on a web site, and view it along a timeline. Something like a central portal for my sms's, call history, social networking updates etc. I should be able to view it along a timeline or for each user. I still havent started to search for a site that would give me such a feature.

Finally found the site that I was looking for

http://www.diigo.com/

Must have songs...

1. Velvet Revolver - Get Out The Door
2. Bush - Head Full Of Ghosts
3. Angels & Airwaves - Secret Crowds
4. mody - When I reach for my revolver
5. Apocalyptica - I'm Not Jesus
6. Louis XIV Finding Out True Love is Blind
7. Tool - Jambi (album ver)
8. Saliva - Doperide
9. Marcy Playground - Saint Joe On A School Bus
10. Disturbed - Inside the Fire ****
11. Disturbed - The Game
12. MorningWood - Nth Degree
13. Cake - Love you Madly
14. Oleander - Why I'm Here
15. The Refreshments - Banditos
16. Big Wrek - The Oaf
17. PearlJam - Go
18. Stroke9 - 100 Girls
19. Guns and Roses - Chinies Democracy
20. Powerman 5000 - When Worlds Collide
21. Eve 6 - Inside Out
22. Monster Magnet - Silver Future
23. Buckcherry - Too Drunk
24. Metallica - The Day that never comes
25. Staind - All I want
26. Shiny Toy Guns - Ghost Town
27. All-American Rejects - Gives You Hell
28. U2 - Get On Your Boots
29. Papa Roach - Lifeline
30. Shiny Toy Guys - Ricochet
31. Thornley - Easy Comes
32. 3 Doors Down - Its not my time
33. Prom Kings - The, Alone
34. Beastie Boys - So What'cha Want
35. Carolina Liar - Show Me What I'm Looking for
36. Collective Soul - Vent
37. Cage The Elephant - Aint No Rest For The Wicked.
38. Loudermilk - Rock N Roll and the Teenage Desperation
39. David Bowie & Trent Reznor - I'm Afraid of Americans
40. Seether - Gasoline
41. Big B - Sinner
42. Lo-Fidelity All-Stars - Battleflag
43. Shinedown - Heroes
44. Limp Bizkit - Nookie
45. Lo-Fidelity All-Stars - Battleflag
46. Kings Of Leon - Notion
47. Cake - Sheep Go To Heaven

Extracting audio from all CFA video files in one command.

Really impressed with myself today.
I have the CFA schwsere CDs with me and these CDs have vidoes in them, but I am not interested in looking at a person talking, I am more interested in listening to the lecture. In short I would like to extract the audio channel for all these videos.
1. Find a tool that extracts audio from the vidoe file - ffmpeg, this tool was able to extract the audio as an mp3 file.
2. Automate the process to go over all the vidoe files and convert them to mp3.

Since all the video files have the same name, and I want the audio file to have diffent names based off the Study Session and Reading, here is what I did
- Find all wmv files
- for each file extract the folder name ( this contains the following format L2_SS18_P7)
- extract the video into an audio file based off the above file name.

find . -name *.wmv -exec sh -c 'ffmpeg -i {} `ls {} | cut -d '/' -f 3`.mp3' \;

Whoo hooo

I found that CD10 and CD11 are the same, so I have to mount and extract them again.


Intermediate commands to test
find . -iname *.flac -exec flac2vorbis {} \; -exec rm -v {} \;


find . -iname *.flac -exec sh -c "flac2vorbis {} && rm {}" \;


find . -name *.wmv -exec sh -c "ls {} | cut -d '/' -f 3" \;


ffmpeg -i ./CD16/L2_SS18_P7/L2_SS18_P7_files/0MM0.wmv `ls ./CD16/L2_SS18_P7/L2_SS18_P7_files/0MM0.wmv | cut -d '/' -f 3`.mp3